From c5a1f5b0e2cfd92cba44d8d860b66151cd267a9e Mon Sep 17 00:00:00 2001 From: Sander Vocke Date: Wed, 29 Jan 2020 11:03:51 +0100 Subject: [PATCH] All-in on SQL.js, simplified. --- src/database.js | 373 ++++--------------------------------------- src/debuggingpage.js | 174 -------------------- src/main.js | 15 +- src/queries.js | 76 ++++----- 4 files changed, 81 insertions(+), 557 deletions(-) delete mode 100644 src/debuggingpage.js diff --git a/src/database.js b/src/database.js index f868885..c6e2c74 100644 --- a/src/database.js +++ b/src/database.js @@ -1,226 +1,17 @@ -import React from 'react'; - -export const DBTypeEnum = { - ALASQL_SQLITE: 1, - ALASQL_INDEXEDDB: 2, - ALASQL_NATIVE: 3, - SQLJS_SQLITE: 4 -}; - -export const DBSourceEnum = { - ATTACHFILE: 1, - CREATE: 2 -} - -export function alasql_async_queries(alasql_object, queries) { - var p = Promise.resolve(null); - for (let i = 0; i < queries.length; i++) { - p = p.then(() => { - return alasql_object.promise(queries[i]); - }); - } - - return p; -} - -export function convert_sqljs_result_to_rows(result_object) { - if(result_object.length == 0) { - return []; - } - - var headers = result_object[0].columns; - var data = result_object[0].values; - var data_out = []; - data.forEach(row => { - var row_out = {}; - for(let i=0; i { - queries.push("CREATE TABLE " + db_name + "." + elem.tableid + ";"); - queries.push("SELECT * INTO " + db_name + "." + elem.tableid + - " FROM " + self.state.db_name + "." + elem.tableid + ";"); - }); - queries.push("DETACH DATABASE " + self.state.db_name + ";"); - queries.push("USE " + db_name + ";"); - self.queries_async(queries).then(() => { - self.state.db_type = db_type; - self.state.db_name = db_name; - resolve(); - }); - }); - } else if (this.state.db_type === DBTypeEnum.ALASQL_SQLITE && db_type === DBTypeEnum.ALASQL_NATIVE) { - return new Promise(function (resolve, reject) { - var tables = self.queries_sync(["SHOW TABLES;"]); - var queries = []; - queries.push("DROP DATABASE IF EXISTS " + db_name + ";"); - queries.push("CREATE DATABASE IF NOT EXISTS " + db_name + ";"); - tables.forEach(elem => { - queries.push("CREATE TABLE " + db_name + "." + elem.tableid + ";"); - queries.push("SELECT * INTO " + db_name + "." + elem.tableid + - " FROM " + self.state.db_name + "." + elem.tableid + ";"); - }); - queries.push("DETACH DATABASE " + self.state.db_name + ";"); - queries.push("USE " + db_name + ";"); - self.queries_async(queries).then(() => { - self.state.db_type = db_type; - self.state.db_name = db_name; - resolve(); - }); - }); - } else if (this.state.db_type === DBTypeEnum.SQLJS_SQLITE && db_type === DBTypeEnum.ALASQL_NATIVE) { - return new Promise(function (resolve, reject) { - var tables = self.queries_sync(["SELECT * FROM sqlite_master WHERE type='table';"])[0].values; - - // Get all table definition lines. - var table_definitions = []; - var table_names = []; - tables.forEach(elem => { - table_definitions.push(elem[4]); - table_names.push(elem[1]); - }); - - // We want to create all these tables in the AlaSQL database as well. However, that will require - // us to escape the table names because AlaSQL may break on colums with names equal to keywords - // (see https://github.com/agershun/alasql/issues/1155). - // Here we take a dirty shortcut: explicitly escape all occurrences of "value", "query" and "matrix". - var new_table_definitions = []; - table_definitions.forEach(elem => { - var newelem = elem - .replace(/value/g, "`value`") - .replace(/query/g, "`query`") - .replace(/matrix/g, "`matrix`"); - new_table_definitions.push(newelem); - }); - - var asql = require('alasql'); - var queries = []; - queries.push("CREATE DATABASE IF NOT EXISTS " + db_name + ";"); - queries.push("USE " + db_name + ";"); - new_table_definitions.forEach(elem => { - queries.push(elem); // Creates the table - }); - alasql_async_queries(asql, queries) - .then(() => { - // Fill the tables by passing through a JS dictionary object. - table_names.forEach(elem => { - var res = self.queries_sync(["SELECT * FROM " + elem + ";"])[0]; - if (res) { - var structured = []; - res.values.forEach(vals => { - var row = {}; - var i = 0; - res.columns.forEach(column => { - row[column] = vals[i]; - i++; - }) - structured.push(row); - }); - asql("SELECT * INTO " + elem + " FROM ?", [structured]); - } - }); - }) - .then(() => { - self.state.db_type = db_type; - self.state.db_name = db_name; - self.state.db_object = asql; - resolve(); - }); - }); - } - throw new Error("Unsupported copy creation from db_type " + this.state.db_type + " to " + db_type); - } -} - -// TODO random database names -function fetch_db_from_sqlite(filename, db_name) { - return new Promise(function (resolve, reject) { - var initSqlJs = require('sql.js'); - initSqlJs({ locateFile: filename => process.env.PUBLIC_URL + `/sql.js/dist/${filename}` }) - .then(SQL => { - var asql = require('alasql'); - asql.promise("ATTACH SQLITE DATABASE " + db_name + "(\"" + filename + "\");") - .then(res => { asql.promise("USE " + db_name + ";"); }) - .then(res => { - var imported_db = new DB(asql, db_name, DBTypeEnum.ALASQL_SQLITE); - resolve(imported_db); - }); - }); - }); -} - -function fetch_sqljs_db_from_sqlite(filename, db_name) { +function fetch_sqljs_db_from_sqlite(filename) { return new Promise(function (resolve, reject) { var initSqlJs = require('sql.js'); initSqlJs({ locateFile: filename => process.env.PUBLIC_URL + `/sql.js/dist/${filename}` }) @@ -235,137 +26,39 @@ function fetch_sqljs_db_from_sqlite(filename, db_name) { .then(data => { var array = new Uint8Array(data); var sqljs_db = new SQL.Database(array); - var imported_db = new DB(sqljs_db, db_name, DBTypeEnum.SQLJS_SQLITE); - resolve(imported_db); + resolve(sqljs_db); }) }); }); } -function create_indexed_db(name) { - return new Promise(function (resolve, reject) { - var asql = require('alasql'); - asql.promise("DROP INDEXEDDB DATABASE " + name + ";") - .then(res => { asql.promise("CREATE INDEXEDDB DATABASE IF NOT EXISTS " + name + ";"); }) - .then(res => { asql.promise("ATTACH INDEXEDDB DATABASE " + name + ";"); }) - //.then(res => { asql.promise("USE " + name + ";"); }) // TODO why does this break things? - .then(res => { - var created_db = new DB(asql, name, DBTypeEnum.ALASQL_INDEXEDDB); - resolve(created_db); - }); - }); +export function regexp_match(string, regex) { + return string.match(regex) != null; } -function create_native_db(name) { - return new Promise(function (resolve, reject) { - var asql = require('alasql'); - asql.promise("DROP DATABASE " + name + ";") - .then(res => { asql.promise("CREATE DATABASE IF NOT EXISTS " + name + ";"); }) - .then(res => { asql.promise("ATTACH DATABASE " + name + ";"); }) - //.then(res => { asql.promise("USE " + name + ";"); }) // TODO why does this break things? - .then(res => { - var created_db = new DB(asql, name, DBTypeEnum.ALASQL_INDEXEDDB); - resolve(created_db); - }); - }); +export function add_custom_functions(db) { + db.create_function("REGEXP", regexp_match); } -export class ProvideDB extends React.Component { - state = { - loading: true, - error: false, - done: false, - db: false +export function ProvideDB(props) { + const { children, db_url } = props; + const [db, setDb] = useState(null); + const [error, setError] = useState(false); + + useEffect(() => { + fetch_sqljs_db_from_sqlite(db_url) + .then(db => { + add_custom_functions(db); + setError(false); + setDb(db); + }) + .catch(error => { setError(error); }); + }, []) + + var child_props = { + db: db, + db_error: error, }; - componentDidMount() { - var output_db_name = (this.props.db_source_type === this.props.db_target_type) ? - this.props.db_source_name : this.props.db_target_name; - if (this.props.db_source_type === DBTypeEnum.ALASQL_SQLITE && this.props.db_source === DBSourceEnum.ATTACHFILE) { - fetch_db_from_sqlite(this.props.db_file, output_db_name) - .then(db => { - if (this.props.db_target_type !== this.props.db_source_type) { - db.migrate_async(this.props.db_target_type, this.props.db_target_name) - .then(() => { this.setState({ loading: false, done: true, db: db }) }); - } else { this.setState({ loading: false, done: true, db: db }) }; - }) - .catch(error => this.setState({ loading: false, done: false, error })); - } else if (this.props.db_source_type === DBTypeEnum.ALASQL_INDEXEDDB && this.props.db_source === DBSourceEnum.CREATE) { - create_indexed_db(output_db_name) - .then(db => { - if (this.props.db_target_type !== this.props.db_source_type) { - db.migrate_async(this.props.db_target_type, this.props.db_target_name) - .then(() => { this.setState({ loading: false, done: true, db: db }) }); - } else { this.setState({ loading: false, done: true, db: db }) }; - }); - } else if (this.props.db_source_type === DBTypeEnum.ALASQL_NATIVE && this.props.db_source === DBSourceEnum.CREATE) { - create_native_db(output_db_name) - .then(db => { - if (this.props.db_target_type !== this.props.db_source_type) { - db.migrate_async(this.props.db_target_type, this.props.db_target_name) - .then(() => { this.setState({ loading: false, done: true, db: db }) }); - } else { this.setState({ loading: false, done: true, db: db }) }; - }); - } else if (this.props.db_source_type === DBTypeEnum.SQLJS_SQLITE && this.props.db_source === DBSourceEnum.ATTACHFILE) { - fetch_sqljs_db_from_sqlite(this.props.db_file, output_db_name) - .then(db => { - if (this.props.db_target_type !== this.props.db_source_type) { - db.migrate_async(this.props.db_target_type, this.props.db_target_name) - .then(() => { this.setState({ loading: false, done: true, db: db }) }); - } else { this.setState({ loading: false, done: true, db: db }) }; - }) - .catch(error => this.setState({ loading: false, done: false, error })); - } else { - throw new Error("Unsupported ProvideDB configuration: from source " + this.props.db_source + - " with source type " + this.props.db_source_type + " to target type " + this.props.db_target_type + "."); - } - } - - render() { - return this.props.children(this.state); - } -} - -export class DBQueryBar extends React.Component { - onChangeHandler = (event) => { - if (this.props.onChange) { this.props.onChange(event.target.value) }; - } - - render() { - return ( -
- -
- ); - } -} - -export class DBQueryConsole extends React.Component { - state = { - processing: false, - result: false, - query: "", - } - - onQueryChangeHandler = query => { this.setState({ query: query }); } - onQuerySubmitHandler = () => { - this.setState({ processing: true, result: false }); - this.props.database.queries_async([this.state.query]) - .then(result => { - this.setState({ processing: false, result: JSON.stringify(result) }); - }); - } - - render() { - return ( - <> -
- - -

Result:

- {this.state.result &&

{this.state.result}

} -
- - ); - } -} + return children({ ...child_props }); +} \ No newline at end of file diff --git a/src/debuggingpage.js b/src/debuggingpage.js deleted file mode 100644 index 56225b5..0000000 --- a/src/debuggingpage.js +++ /dev/null @@ -1,174 +0,0 @@ -import React from 'react'; -import { Fetch } from './fetch.js'; -import { ProvideDB, DBQueryConsole, DBQueryBar, DBTypeEnum, DBSourceEnum } from './database.js'; -import { PhotoTableLine, MediaTableLine } from './media.js'; -import { do_image_query, do_album_query, image_query_with_where, user_query_from_search_string, maybe_image_query, maybe_album_query } from './queries.js'; - -const URLLoading = ({ url }) =>

Loading: {url}

; -const URLError = ({ error }) =>

Failed to load URL resource: {error.message}

; -const URLFinished = ({ url, data }) =>

{url} was loaded: {data.byteLength} bytes received.

; - -const DBLoading = ({ sqlite_file }) =>

Loading: {sqlite_file}

; -const DBError = ({ error }) =>

Failed to load database: {error.message}

; -const DBFinished = ({ sqlite_file, db }) =>

{sqlite_file} was loaded. Name: {db.state.db_name}.

; - -const TestFetch = ({ url }) => ( - - {({ loading, error, done, data }) => ( - <> - {loading && } - {error && } - {done && } - - )} - -) - -export class PhotoFromDB extends React.Component { - state = { - done: false, - photo: false, - } - - componentDidMount() { - do_image_query(this.props.sql_query, this.props.database, "/test_photos", "/test_photos_thumbs").then(photos => { - this.setState({ done: true, photo: photos[0] }); - }); - } - - render() { - return ( - <> - {!this.state.done &&

>Querying photo from DB: {this.props.sql_query}

} - {this.state.photo && } - - ); - } -} - -export class ImageWhereConsole extends React.Component { - state = { - processing: false, - result: false, - where: "", - } - - onQueryChangeHandler = whereclause => { this.setState({ where: whereclause }); } - onQuerySubmitHandler = () => { - this.setState({ processing: true, result: false }); - this.props.database.queries_async([image_query_with_where(this.state.where, this.props.database)]) - .then(result => { - this.setState({ processing: false, result: JSON.stringify(result) }); - }); - } - - render() { - return ( - <> -
- - -

Result:

- {this.state.result &&

{this.state.result}

} -
- - ); - } -} - -const MediaTable = ({ items }) => ( - <> - { - items.map((value, index) => { - return value && - }) - } - -) - -export class TestDBStringQuery extends React.Component { - state = { - string: false, - query: false, - sql_image_query: false, - sql_album_query: false, - photos: false, - albums: false, - } - - onQueryChangeHandler = str => { this.setState({ string: str }); } - onQuerySubmitHandler = () => { - var q = user_query_from_search_string(this.state.string); - var sql_image_query = maybe_image_query(q, this.props.db); - var sql_album_query = maybe_album_query(q, this.props.db); - this.setState({ query: q, sql_image_query: sql_image_query, sql_album_query: sql_album_query, photos: false, albums: false }); - do_image_query(sql_image_query, this.props.db, "/test_photos", "/test_photos_thumbs").then(photos => { - this.setState({ done: true, photos: photos }); - }); - do_album_query(sql_album_query, this.props.db, "/test_photos", "/test_photos_thumbs").then(albums => { - this.setState({ done: true, albums: albums }); - }); - } - - render() { - - return ( - <> -
- - - { this.state.photos &&

Found {this.state.photos.length} photos.

} - { this.state.photos && } - { this.state.albums &&

Found {this.state.albums.length} albums.

} - { this.state.albums && } -
- - ); - } -} - -const TestDBFetch = ({ db }) => ( - <> -

DB Query Console

- -

Example photo from DB

- -

DB WHERE clause image search

- - -) - -const TestDBPlayground = ({ db }) => ( - <> - - -) - -export const DebuggingPage = () => ( - <> -

Test file fetching:

- - - {({ loading, error, done, db }) => ( - - <> - {loading && } - {error && } - {done && ( - <> - -

IndexedDB playground:

- -

Test DB fetching:

- -

Test queries:

- - - ) - } - - )} - - -); diff --git a/src/main.js b/src/main.js index 5fbaba7..b8d6c5d 100644 --- a/src/main.js +++ b/src/main.js @@ -7,7 +7,7 @@ import { createMuiTheme, ThemeProvider } from '@material-ui/core/styles'; import { SearchBar } from './searchbar.js'; import { InternalErrorPage } from './error.js'; import { LoadingPage } from './loading.js'; -import { ProvideDB, DBTypeEnum, DBSourceEnum } from './database.js'; +import { ProvideDB } from './database.js'; import { UserQuery, user_query_from_search_string, maybe_image_query, do_image_query, maybe_album_query, do_album_query, maybe_tag_query, do_tag_query, ConstFilter, ResultTypeEnum } from './queries.js'; import { Browser } from './browser.js'; import { UserQueryWidget } from './userquerywidget.js'; @@ -148,14 +148,15 @@ const theme = createMuiTheme({ export function MainPage() { return ( - - {({ loading, error, done, db }) => ( + + {({ db_error, db }) => ( <> - {loading && } - {error && } - {done && } + {(db == null && !db_error) && } + {db_error && } + {db != null && } )} diff --git a/src/queries.js b/src/queries.js index b9d034a..7103530 100644 --- a/src/queries.js +++ b/src/queries.js @@ -1,6 +1,6 @@ import { create_photo, create_album, create_tag } from './media.js'; -import { DBTypeEnum } from './database.js'; +import { sqljs_async_queries } from './database.js'; export function escape_regex(s) { return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); @@ -10,13 +10,23 @@ export function do_image_query(query, database, collection_path, collection_thum return new Promise(function (resolve, reject) { var queries = []; queries.push(query); - database.queries_async(queries).then(res => { + sqljs_async_queries(database, queries).then(res => { var photos = []; - if (res && Array.isArray(res)) { - res.forEach(row => { - var imagepath = process.env.PUBLIC_URL + collection_path + "/" + row["relativePath"] + "/" + row["name"]; - var thumbpath = process.env.PUBLIC_URL + collection_thumbs_path + "/" + row["uniqueHash"] + ".jpg"; - photos.push(create_photo(row["id"], row["name"], imagepath, thumbpath, [row["width"], row["height"]])); + if (res && Array.isArray(res) && res.length > 0) { + var cols = res[0].columns; + var data = res[0].values; + data.forEach(row => { + var imagepath = process.env.PUBLIC_URL + collection_path + "/" + row[cols.indexOf("relativePath")] + "/" + row[cols.indexOf("name")]; + var thumbpath = process.env.PUBLIC_URL + collection_thumbs_path + "/" + row[cols.indexOf("uniqueHash")] + ".jpg"; + photos.push( + create_photo( + row[cols.indexOf("id")], + row[cols.indexOf("name")], + imagepath, + thumbpath, + [row[cols.indexOf("width")], + row[cols.indexOf("height")]] + )); }); } resolve(photos); @@ -28,12 +38,15 @@ export function do_album_query(query, database) { return new Promise(function (resolve, reject) { var queries = []; queries.push(query); - database.queries_async(queries).then(res => { + + sqljs_async_queries(database, queries).then(res => { var albums = []; - if (res && Array.isArray(res)) { - res.forEach(row => { - var album_name = row["relativePath"].substring(row["relativePath"].lastIndexOf('/') + 1); - albums.push(create_album(row["id"], album_name, row["relativePath"])); + if (res && Array.isArray(res) && res.length > 0) { + var cols = res[0].columns; + var data = res[0].values; + data.forEach(row => { + var album_name = row[cols.indexOf("relativePath")].substring(row[cols.indexOf("relativePath")].lastIndexOf('/') + 1); + albums.push(create_album(row[cols.indexOf("id")], album_name, row[cols.indexOf("relativePath")])); }); } resolve(albums); @@ -45,11 +58,13 @@ export function do_tag_query(query, database) { return new Promise(function (resolve, reject) { var queries = []; queries.push(query); - database.queries_async(queries).then(res => { + sqljs_async_queries(database, queries).then(res => { var tags = []; - if (res && Array.isArray(res)) { - res.forEach(row => { - tags.push(create_tag(row["id"], row["name"])); + if (res && Array.isArray(res) && res.length > 0) { + var cols = res[0].columns; + var data = res[0].values; + data.forEach(row => { + tags.push(create_tag(row[cols.indexOf("id")], row[cols.indexOf("name")])); }); } resolve(tags); @@ -145,19 +160,19 @@ export class MatchingFilter extends ResultFilter { if (this.match_type == MatchTypeEnum.MATCH_IMAGE_NAME_EQUALS) { return '(Images.name="' + this.match_from + '")'; } else if (this.match_type == MatchTypeEnum.MATCH_IMAGE_NAME_NATURAL) { - return '(LOWER(Images.name) REGEXP LOWER(".*' + return 'REGEXP(LOWER(Images.name), LOWER(".*' + escape_regex(this.match_from) + '.*"))'; } else if (this.match_type == MatchTypeEnum.MATCH_ALBUM_EQUALS) { return '(Albums.relativePath = "' + this.match_from + '")'; } else if (this.match_type == MatchTypeEnum.MATCH_ALBUM_EQUALS_OR_CHILD) { - return '(Albums.relativePath REGEXP "' + return 'REGEXP(Albums.relativePath, "' + escape_regex(this.match_from) + '(\/[^\/]+)*")'; } else if (this.match_type == MatchTypeEnum.MATCH_ALBUM_NATURAL) { throw new Error("Natural matching on album names is not yet supported."); } else if (this.match_type == MatchTypeEnum.MATCH_ALBUM_NAME_EQUALS) { - return '(Albums.relativePath REGEXP "\/(.*\/)*' + return 'REGEXP(Albums.relativePath, "\/(.*\/)*' + escape_regex(this.match_from) + '(\/[^\/]+)*")'; } else if (this.match_type == MatchTypeEnum.MATCH_TAG_EQUALS) { @@ -226,34 +241,23 @@ export class UserQuery { } // This query will return database entries with the fields "id", "uniqueHash", "relativePath" (of the album) and "name" for each matching image. -export function image_query_with_where(maybe_where, db) { - var base_query = "SELECT Images.id, Images.name, Images.uniqueHash, Albums.relativePath, " +export function image_query_with_where(maybe_where) { + return "SELECT Images.id, Images.name, Images.uniqueHash, Albums.relativePath, " + "ImageInformation.width, ImageInformation.height " + "FROM Images INNER JOIN Albums ON Images.album=Albums.id " + "LEFT JOIN ImageTags ON Images.id=ImageTags.imageid " + "LEFT JOIN ImageInformation ON Images.id=ImageInformation.imageid " - + "LEFT JOIN Tags ON ImageTags.tagid=Tags.id " + (maybe_where ? maybe_where : ""); - - if(db.state.db_type == DBTypeEnum.SQLJS_SQLITE) { - return base_query + " GROUP BY Images.id"; - } else if(db.state.db_type == DBTypeEnum.ALASQL_NATIVE || - db.state.db_type == DBTypeEnum.ALASQL_SQLITE || - db.state.db_type == DBTypeEnum.ALASQL_INDEXEDDB) { - // TODO : the GROUP BY used above for some reason breaks AlaSQL. - // For the time being, omit it at the cost of duplicate results. - return base_query; - } - - throw new Error('Unsupported database type for image query: ' + db); + + "LEFT JOIN Tags ON ImageTags.tagid=Tags.id " + (maybe_where ? maybe_where : "") + + " GROUP BY Images.id;"; } // This query will return database entries with the fields "id" and "relativePath" for each matching album. -export function album_query_with_where(maybe_where, db) { +export function album_query_with_where(maybe_where) { return "SELECT Albums.id, Albums.relativePath FROM Albums " + (maybe_where ? maybe_where : "") + ";"; } // This query will return database entries with the fields "id" and "name" for each matching tag. -export function tag_query_with_where(maybe_where, db) { +export function tag_query_with_where(maybe_where) { return "SELECT Tags.id, Tags.name FROM Tags " + (maybe_where ? maybe_where : "") + ";"; }