diff --git a/Cargo.lock b/Cargo.lock index cc1e1f0..7e31811 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,26 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "actix-files" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51e8a9146c12fce92a6e4c24b8c4d9b05268130bfd8d61bc587e822c32ce689" +dependencies = [ + "actix-service", + "actix-web", + "bitflags", + "bytes 0.5.6", + "derive_more", + "futures-core", + "futures-util", + "log", + "mime", + "mime_guess", + "percent-encoding", + "v_htmlescape", +] + [[package]] name = "actix-http" version = "2.2.1" @@ -386,6 +406,15 @@ dependencies = [ "libc", ] +[[package]] +name = "buf-min" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa17aa1cf56bdd6bb30518767d00e58019d326f3f05d8c3e0730b549d332ea83" +dependencies = [ + "bytes 0.5.6", +] + [[package]] name = "bumpalo" version = "3.8.0" @@ -472,7 +501,7 @@ checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "percent-encoding", "time", - "version_check", + "version_check 0.9.3", ] [[package]] @@ -696,7 +725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" dependencies = [ "typenum", - "version_check", + "version_check 0.9.3", ] [[package]] @@ -946,6 +975,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1009,6 +1048,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check 0.1.5", +] + [[package]] name = "num" version = "0.4.0" @@ -1490,6 +1539,7 @@ dependencies = [ name = "sqlite_database_file_dissect" version = "0.1.0" dependencies = [ + "actix-files", "actix-web", "clap", "hex", @@ -1518,7 +1568,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" dependencies = [ - "version_check", + "version_check 0.9.3", ] [[package]] @@ -1636,7 +1686,7 @@ dependencies = [ "standback", "stdweb", "time-macros", - "version_check", + "version_check 0.9.3", "winapi 0.3.9", ] @@ -1804,6 +1854,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.3", +] + [[package]] name = "unicode-bidi" version = "0.3.7" @@ -1849,12 +1908,50 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "v_escape" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e0ab5fab1db278a9413d2ea794cb66f471f898c5b020c3c394f6447625d9d4" +dependencies = [ + "buf-min", + "v_escape_derive", +] + +[[package]] +name = "v_escape_derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29769400af8b264944b851c961a4a6930e76604f59b1fcd51246bab6a296c8c" +dependencies = [ + "nom", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "v_htmlescape" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9a8af610ad6f7fc9989c9d2590d9764bc61f294884e9ee93baa58795174572" +dependencies = [ + "cfg-if 1.0.0", + "v_escape", +] + [[package]] name = "vec_map" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + [[package]] name = "version_check" version = "0.9.3" diff --git a/Cargo.toml b/Cargo.toml index 9b2a19d..dd58a70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ serde_derive = "1.0.129" actix-web = "3" clap = "2.33.3" lazy_static = "1.4.0" +actix-files = "0.5.0" [profile.dev] opt-level = 0 diff --git a/README.md b/README.md new file mode 100644 index 0000000..92a74c6 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Sqlite database file structure visualization + +Basically, it dissects sqlite database btree file, and visulize the btree node(page) relationship and node(page) content. + +## Run the application + +- http_main --file test-data/Chinbook.db.4.analyze + +## Visualization + +- Open http://127.0.0.1:8080/static/index.html +- Move mouse around btree node(page), it shows parent node in green, and its children nodes in red. +- Mouse left click to show node(page) content in json format. +- Mouse right click to freeze the page relation regardless mouse move. + +## Sample +![Sample](static/sqlite_database_file_dissect.gif) \ No newline at end of file diff --git a/src/components/database_header.rs b/src/components/database_header.rs index b407792..9bbc426 100644 --- a/src/components/database_header.rs +++ b/src/components/database_header.rs @@ -1,3 +1,5 @@ +use serde_derive::Serialize; + use num_derive::FromPrimitive; use num_derive::ToPrimitive; @@ -5,13 +7,13 @@ use crate::utils::error::MyError; use crate::utils::error::ErrorKind; use crate::utils::convert::TryFromBytes; -#[derive(Debug, FromPrimitive, ToPrimitive)] +#[derive(Debug, FromPrimitive, ToPrimitive, Serialize)] pub enum FileFormatVersion { Legacy = 1, WAL = 2, } -#[derive(Debug, FromPrimitive, ToPrimitive)] +#[derive(Debug, FromPrimitive, ToPrimitive, Serialize)] pub enum SchemaFormatNumber { One = 1, Two = 2, @@ -19,14 +21,14 @@ pub enum SchemaFormatNumber { Four = 4, } -#[derive(Debug, FromPrimitive, ToPrimitive)] +#[derive(Debug, FromPrimitive, ToPrimitive, Serialize)] pub enum TextEncoding { UTF8 = 1, UTF16le = 2, UTF16be = 3, } -#[derive(Debug)] +#[derive(Debug, Serialize)] pub struct DatabaseHeader { pub header_string: String, pub page_size: u16, diff --git a/src/http_main.rs b/src/http_main.rs index 1924f6c..0ae082e 100644 --- a/src/http_main.rs +++ b/src/http_main.rs @@ -9,10 +9,10 @@ use std::fs::File; use std::convert::TryInto; use actix_web::{get, web, App as WebApp, HttpResponse, HttpServer, Responder}; +use actix_files as fs; use clap::{Arg, App as ClapApp}; use serde_json::json; use lazy_static::*; -use sqlite_database_file_dissect::components::database_header; use std::sync::Mutex; use std::sync::MutexGuard; @@ -141,7 +141,18 @@ async fn btree_page_num() -> impl Responder { }); HttpResponse::Ok().body(r) - +} + +#[get("/database_header")] +async fn fetch_database_header() -> impl Responder { + let mut f = File::open(SQLITE_DATABASE_FILE.lock().unwrap().as_ref().unwrap()).unwrap(); + let mut buffer = vec![0 as u8; 100]; + let _r = f.read(&mut buffer); + let database_header = DatabaseHeader::try_from_be_bytes(&buffer[0..100]).unwrap(); + + let r = serde_json::to_string(&database_header).unwrap(); + + HttpResponse::Ok().body(r) } #[get("/btree_page/{page_index}")] @@ -151,7 +162,7 @@ async fn btree_page(web::Path(page_index): web::Path) -> impl Responder { let f_length: usize = f.metadata().unwrap().len().try_into().unwrap(); let page_num: usize = f_length/page_size; - if page_index <= 0 || page_index > page_num { + if page_index < 0 || page_index > page_num { let r = serde_json::to_string( &HttpError::new(HttpErrorKind::PageIndexError(page_index)) ).unwrap(); @@ -210,9 +221,11 @@ async fn main() -> std::io::Result<()>{ HttpServer::new(|| { WebApp::new() + .service(fs::Files::new("/static", "./static").show_files_listing()) .service(btree_hierachy) .service(btree_page) .service(btree_page_num) + .service(fetch_database_header) }) .bind("127.0.0.1:8080")? .run() diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..16b93f0 --- /dev/null +++ b/static/index.html @@ -0,0 +1,128 @@ + + + + + + + + + + + +
+

+

+    
+
+ +
+
+
+ + diff --git a/static/jquery.json-viewer.css b/static/jquery.json-viewer.css new file mode 100644 index 0000000..57aa450 --- /dev/null +++ b/static/jquery.json-viewer.css @@ -0,0 +1,57 @@ +/* Root element */ +.json-document { + padding: 1em 2em; +} + +/* Syntax highlighting for JSON objects */ +ul.json-dict, ol.json-array { + list-style-type: none; + margin: 0 0 0 1px; + border-left: 1px dotted #ccc; + padding-left: 2em; +} +.json-string { + color: #0B7500; +} +.json-literal { + color: #1A01CC; + font-weight: bold; +} + +/* Toggle button */ +a.json-toggle { + position: relative; + color: inherit; + text-decoration: none; +} +a.json-toggle:focus { + outline: none; +} +a.json-toggle:before { + font-size: 1.1em; + color: #c0c0c0; + content: "\25BC"; /* down arrow */ + position: absolute; + display: inline-block; + width: 1em; + text-align: center; + line-height: 1em; + left: -1.2em; +} +a.json-toggle:hover:before { + color: #aaa; +} +a.json-toggle.collapsed:before { + /* Use rotated down arrow, prevents right arrow appearing smaller than down arrow in some browsers */ + transform: rotate(-90deg); +} + +/* Collapsable placeholder links */ +a.json-placeholder { + color: #aaa; + padding: 0 1em; + text-decoration: none; +} +a.json-placeholder:hover { + text-decoration: underline; +} diff --git a/static/jquery.json-viewer.js b/static/jquery.json-viewer.js new file mode 100644 index 0000000..611411b --- /dev/null +++ b/static/jquery.json-viewer.js @@ -0,0 +1,158 @@ +/** + * jQuery json-viewer + * @author: Alexandre Bodelot + * @link: https://github.com/abodelot/jquery.json-viewer + */ +(function($) { + + /** + * Check if arg is either an array with at least 1 element, or a dict with at least 1 key + * @return boolean + */ + function isCollapsable(arg) { + return arg instanceof Object && Object.keys(arg).length > 0; + } + + /** + * Check if a string represents a valid url + * @return boolean + */ + function isUrl(string) { + var urlRegexp = /^(https?:\/\/|ftps?:\/\/)?([a-z0-9%-]+\.){1,}([a-z0-9-]+)?(:(\d{1,5}))?(\/([a-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)?)?$/i; + return urlRegexp.test(string); + } + + /** + * Transform a json object into html representation + * @return string + */ + function json2html(json, options) { + var html = ''; + if (typeof json === 'string') { + // Escape tags and quotes + json = json + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/'/g, ''') + .replace(/"/g, '"'); + + if (options.withLinks && isUrl(json)) { + html += '' + json + ''; + } else { + // Escape double quotes in the rendered non-URL string. + json = json.replace(/"/g, '\\"'); + html += '"' + json + '"'; + } + } else if (typeof json === 'number') { + html += '' + json + ''; + } else if (typeof json === 'boolean') { + html += '' + json + ''; + } else if (json === null) { + html += 'null'; + } else if (json instanceof Array) { + if (json.length > 0) { + html += '[
    '; + for (var i = 0; i < json.length; ++i) { + html += '
  1. '; + // Add toggle button if item is collapsable + if (isCollapsable(json[i])) { + html += ''; + } + html += json2html(json[i], options); + // Add comma if item is not last + if (i < json.length - 1) { + html += ','; + } + html += '
  2. '; + } + html += '
]'; + } else { + html += '[]'; + } + } else if (typeof json === 'object') { + var keyCount = Object.keys(json).length; + if (keyCount > 0) { + html += '{}'; + } else { + html += '{}'; + } + } + return html; + } + + /** + * jQuery plugin method + * @param json: a javascript object + * @param options: an optional options hash + */ + $.fn.jsonViewer = function(json, options) { + // Merge user options with default options + options = Object.assign({}, { + collapsed: false, + rootCollapsable: true, + withQuotes: false, + withLinks: true + }, options); + + // jQuery chaining + return this.each(function() { + + // Transform to HTML + var html = json2html(json, options); + if (options.rootCollapsable && isCollapsable(json)) { + html = '' + html; + } + + // Insert HTML in target DOM element + $(this).html(html); + $(this).addClass('json-document'); + + // Bind click on toggle buttons + $(this).off('click'); + $(this).on('click', 'a.json-toggle', function() { + var target = $(this).toggleClass('collapsed').siblings('ul.json-dict, ol.json-array'); + target.toggle(); + if (target.is(':visible')) { + target.siblings('.json-placeholder').remove(); + } else { + var count = target.children('li').length; + var placeholder = count + (count > 1 ? ' items' : ' item'); + target.after('' + placeholder + ''); + } + return false; + }); + + // Simulate click on toggle button when placeholder is clicked + $(this).on('click', 'a.json-placeholder', function() { + $(this).siblings('a.json-toggle').click(); + return false; + }); + + if (options.collapsed == true) { + // Trigger click to collapse all nodes + $(this).find('a.json-toggle').click(); + } + }); + }; +})(jQuery); diff --git a/static/sqlite_database_file_dissect.gif b/static/sqlite_database_file_dissect.gif new file mode 100644 index 0000000..6232dd9 Binary files /dev/null and b/static/sqlite_database_file_dissect.gif differ