Digikam-FatClient/src/database.js

import React, { useEffect, useState, useContext } from 'react';
import NodeEnvironment from 'jest-environment-node';

import {
  add_geo_area_to_store, get_geo_area_from_store,
  initialize_image_index, image_in_area
} from './geo_store.js';

import * as turf from '@turf/turf'
import KDBush from 'kdbush';

export async function sqljs_async_queries(sqljs_object, queries) {
  //var t0 = performance.now();
  for (let i = 0; i < (queries.length - 1); i++) {
    //console.log("Query: ", queries[i]);
    try {
      sqljs_object.exec(queries[i]);
    } catch (e) {
      throw e;
    }
  }
  //console.log("Query: ", queries[queries.length - 1]);
  try {
    var r = sqljs_object.exec(queries[queries.length - 1]);
  } catch (e) {
    throw e;
  }
  //console.log("Queries took ", (performance.now() - t0),  " ms.");
  //console.log("Query result for ", queries[queries.length - 1], ": ", r);
  return r;
}

function fetch_sqljs_db_from_sqlite(filename) {
  return new Promise(function (resolve, reject) {
    var initSqlJs = require('sql.js');
    initSqlJs({ locateFile: filename => process.env.PUBLIC_URL + `/sql.js/dist/${filename}` })
      .then(SQL => {
        fetch(filename)
          .then(res => {
            if (!res.ok) {
              throw new Error(res.status);
            }
            return res.arrayBuffer();
          })
          .then(data => {
            var array = new Uint8Array(data);
            var sqljs_db = new SQL.Database(array);
            resolve(sqljs_db);
          })
      });
  });
}

export function regexp_match(string, regex) {
  return string.match(regex) != null;
}

export function is_in_geo_polygon_from_store(image_id, lat, long, polygon_hash) {
  return image_in_area(image_id, polygon_hash);
}

// Digikam stores its tree of tags as individual tags,
// linked only by their parent ID. This makes searching
// difficult. Therefore we add a column in the tag table
// which holds the "full tag" (e.g. People/John for the tag John).
export async function add_full_tag_info(db) {
  var res = db.exec("SELECT id, pid, name FROM Tags LEFT JOIN TagProperties ON Tags.id=TagProperties.tagid");
  if (!Array.isArray(res) || res.length == 0) {
    throw new Error("Couldn't get tags information.");
  }

  var cols = res[0].columns;
  var all_tags = res[0].values;
  var full_tags = []; // id, full name pairs

  const get_tag_by_id = (id) => {
    for (let i = 0; i < all_tags.length; i++) {
      if (all_tags[i][cols.indexOf("id")] == id) {
        return all_tags[i];
      }
    }
    return null;
  };

  // Solve the full paths for each tag.
  all_tags.forEach(row => {
    var name = row[cols.indexOf("name")];
    var full = name;
    var pid = row[cols.indexOf("pid")];
    var id = row[cols.indexOf("id")];
    var parent = pid == 0 ? null : get_tag_by_id(pid);
    while (parent != null) {
      full = parent[cols.indexOf("name")] + "/" + full;
      pid = parent[cols.indexOf("pid")];
      parent = pid == 0 ? null : get_tag_by_id(pid);
    }
    var new_entry = { id: id, fullname: full };
    full_tags.push(new_entry);
  });

  // Now put them in the database.
  db.exec("ALTER TABLE Tags ADD fullname TEXT;");
  full_tags.forEach(tag => {
    var query = 'UPDATE Tags SET fullname="' + tag.fullname + '" WHERE id=' + tag.id + ';';
    db.exec(query);
  });

  return db;
}

function polygons_benchmark(database) {
  var img_query = "SELECT GROUP_CONCAT(Images.id), ImagePositions.latitudeNumber, ImagePositions.longitudeNumber FROM Images "
    + "LEFT JOIN ImagePositions ON ImagePositions.imageid=Images.id WHERE ImagePositions.latitudeNumber NOT NULL GROUP BY ImagePositions.latitudeNumber, ImagePositions.longitudeNumber;";

  sqljs_async_queries(database, [img_query]).then(res => {
    console.log("Images query result:", res);

    fetch("https://nominatim.openstreetmap.org/search?polygon_geojson=1&polygon_threshold=0.001&format=json&limit=1&q=Australia")
      .then(res => res.json())
      .then(jsonres => {
        var polies;
        var points = []; // Contains all unique geometric points as [lat long]
        var ids_per_point = []; // Contains a list of ids for each unique geometric point

        console.log("Nominatim geo answer:", jsonres);

        if (Array.isArray(jsonres) && jsonres.length > 0) {
          polies = jsonres[0].geojson;
        }

        if (res && Array.isArray(res) && res.length > 0) {
          var cols = res[0].columns;
          var data = res[0].values;
          data.forEach(row => {
            points.push([parseFloat(row[cols.indexOf("longitudeNumber")]), parseFloat(row[cols.indexOf("latitudeNumber")])]);
            ids_per_point.push(row[cols.indexOf("GROUP_CONCAT(Images.id)")].split(',').map(e => parseInt(e)));
          });
        }

        console.log("Points: ", points);
        console.log("IDs per point: ", ids_per_point);
        console.log("Nominatim GEOJSON: ", polies);

        {
          // Try Turf
          const tpoints = turf.points(points);
          console.time("Turf points within polygon");
          var found = turf.pointsWithinPolygon(tpoints, polies);
          console.timeEnd("Turf points within polygon");
          console.log("Turf PointsWithinPolygon: ", found);
        }

        {
          // Try KDBush
          console.time("Build KDBush index");
          const index = new KDBush(points, p => p[0], p => p[1], 16, Float64Array);
          console.timeEnd("Build KDBush index");

          console.time("Total KDBush search time");

          // Get bounding boxes for all subpolygons
          var boxes = [];
          console.time("Build KDBush poly boxes");
          for (let i = 0; i < polies.coordinates.length; i++) {
            const outerPoly = polies.coordinates[i][0];
            var minx = Number.POSITIVE_INFINITY;
            var miny = Number.POSITIVE_INFINITY;
            var maxx = Number.NEGATIVE_INFINITY;
            var maxy = Number.NEGATIVE_INFINITY;
            for (let j = 0; j < outerPoly.length; j++) {
              minx = Math.min(minx, outerPoly[j][0]);
              miny = Math.min(miny, outerPoly[j][1]);
              maxx = Math.max(maxx, outerPoly[j][0]);
              maxy = Math.max(maxy, outerPoly[j][1]);
            }
            boxes.push([minx, miny, maxx, maxy]);
          }
          console.timeEnd("Build KDBush poly boxes");
          console.log("KDBush boxes: ", boxes);

          // Test points in KD tree against each subpolygon bounding box
          console.time("Find box points in KDBush");
          let hits = new Set();
          for (let i = 0; i < boxes.length; i++) {
            const ids = index.range(boxes[i][0], boxes[i][1], boxes[i][2], boxes[i][3]);
            ids.forEach(e => hits.add(e));
          }
          console.timeEnd("Find box points in KDBush");
          console.log("Hits: ", hits);

          // Test hit points exactly
          console.time("Get exact hits after KDBush");
          var realhits = [];
          hits.forEach(hit => {
            const point = points[hit];
            const is_real_hit = turf.booleanPointInPolygon(turf.point(points[hit]), polies);
            if (is_real_hit) {
              realhits.push(hit);
            }
          });
          console.log("Real: ", realhits);
          console.timeEnd("Get exact hits after KDBush");

          // Expand to all image IDs
          console.time("Expand exact hits to IDs");
          var hit_ids = [];
          realhits.forEach(idx => {
            hit_ids = hit_ids.concat(ids_per_point[idx]);
          });
          console.log("Real hit IDs: ", hit_ids);
          console.timeEnd("Expand exact hits to IDs");

          console.timeEnd("Total KDBush search time");
        }
      });
  })
    .catch(err => { throw err; });
}

export function ProvideDB(props) {
  const { children, db_url } = props;
  const [db, setDb] = useState(null);
  const [error, setError] = useState(false);

  useEffect(() => {
    fetch_sqljs_db_from_sqlite(db_url)
      .then(db => {
        add_full_tag_info(db)
          .then((newdb) => {
            initialize_image_index(newdb).then(() => {
              db.create_function("REGEXP", regexp_match);
              db.create_function("IS_IN_GEO", is_in_geo_polygon_from_store);
              //polygons_benchmark(db);
              setError(false);
              setDb(newdb);
            })
          })
      })
      .catch(error => { setError(error); });
  }, [])

  var child_props = {
    db: db,
    db_error: error,
  };

  return children({ ...child_props });
}