Basic search implemented.

client/package-lock.json

@@ -2066,6 +2066,11 @@
         }
       }
     },
+    "@types/tiny-async-pool": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/tiny-async-pool/-/tiny-async-pool-1.0.0.tgz",
+      "integrity": "sha512-d8RK1jg/piCgv5/jD8ta8uJOE10tU8MWExzL1Kf1kOjMaTuL5cW0eZ9ax001SSYa4Ecg6xzZBh/jM4GB7+5OAg=="
+    },
     "@types/uuid": {
       "version": "8.3.0",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz",
@@ -13285,6 +13290,22 @@
       "resolved": "https://registry.npmjs.org/timsort/-/timsort-0.3.0.tgz",
       "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q="
     },
+    "tiny-async-pool": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.2.0.tgz",
+      "integrity": "sha512-PY/OiSenYGBU3c1nTuP1HLKRkhKFDXsAibYI5GeHbHw2WVpt6OFzAPIRP94dGnS66Jhrkheim2CHAXUNI4XwMg==",
+      "requires": {
+        "semver": "^5.5.0",
+        "yaassertion": "^1.0.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
     "tiny-invariant": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.1.0.tgz",
@@ -14585,6 +14606,11 @@
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
     },
+    "yaassertion": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
+      "integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
+    },
     "yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

client/package.json

@@ -15,6 +15,7 @@
     "@types/react-dom": "^16.9.0",
     "@types/react-router": "^5.1.8",
     "@types/react-router-dom": "^5.1.5",
+    "@types/tiny-async-pool": "^1.0.0",
     "@types/uuid": "^8.3.0",
     "jsurl": "^0.1.5",
     "lodash": "^4.17.20",
@@ -27,6 +28,7 @@
     "react-error-boundary": "^3.0.2",
     "react-router-dom": "^5.2.0",
     "react-scripts": "^3.4.3",
+    "tiny-async-pool": "^1.2.0",
     "ts-enum-util": "^4.0.2",
     "typescript": "~3.7.2",
     "uuid": "^8.3.0"

client/src/api.ts

@@ -67,6 +67,7 @@ export const QueryEndpoint = '/query';
 export enum QueryElemOp {
     And = "AND",
     Or = "OR",
+    Not = "NOT",
 }
 export enum QueryFilterOp {
     Eq = "EQ",

client/src/components/windows/manage_links/BatchLinkDialog.tsx

@@ -1,9 +1,16 @@
-import React, { useState } from 'react';
+import React, { Dispatch, SetStateAction, useEffect, useState } from 'react';
 import { Box, Button, Checkbox, createStyles, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, Divider, FormControlLabel, List, ListItem, ListItemIcon, ListItemText, makeStyles, MenuItem, Paper, Select, Theme, Typography } from "@material-ui/core";
 import StoreLinkIcon from '../../common/StoreLinkIcon';
 import { $enum } from 'ts-enum-util';
 import { IntegrationState, useIntegrations } from '../../../lib/integration/useIntegrations';
-import { ExternalStore, IntegrationStores, IntegrationType } from '../../../api';
+import { ExternalStore, IntegrationStores, IntegrationType, ItemType, QueryResponseType, StoreURLIdentifiers } from '../../../api';
+import { start } from 'repl';
+import { QueryLeafBy, QueryLeafOp, QueryNodeOp, queryNot } from '../../../lib/query/Query';
+import { queryAlbums, queryArtists, queryItems, querySongs } from '../../../lib/backend/queries';
+import asyncPool from "tiny-async-pool";
+import { getSong } from '../../../lib/backend/songs';
+import { getAlbum } from '../../../lib/backend/albums';
+import { getArtist } from '../../../lib/backend/artists';
 
 const useStyles = makeStyles((theme: Theme) =>
     createStyles({
@@ -13,6 +20,192 @@ const useStyles = makeStyles((theme: Theme) =>
     })
 );
 
+enum BatchJobState {
+    Idle = 0,
+    Collecting,
+    Running,
+}
+
+interface Task {
+    itemType: ItemType,
+    itemId: number,
+    integrationId: number,
+    store: ExternalStore,
+}
+
+interface BatchJobStatus {
+    state: BatchJobState,
+    numTasks: number,
+    tasksSuccess: number,
+    tasksFailed: number,
+}
+
+async function makeTasks(
+    integration: IntegrationState,
+    linkSongs: boolean,
+    linkArtists: boolean,
+    linkAlbums: boolean,
+    addTaskCb: (t: Task) => void,
+) {
+    let whichProp: any = {
+        [ItemType.Song]: QueryLeafBy.SongStoreLinks,
+        [ItemType.Artist]: QueryLeafBy.ArtistStoreLinks,
+        [ItemType.Album]: QueryLeafBy.AlbumStoreLinks,
+    }
+    let whichElem: any = {
+        [ItemType.Song]: 'songs',
+        [ItemType.Artist]: 'artists',
+        [ItemType.Album]: 'albums',
+    }
+    let maybeStore = integration.integration.providesStoreLink();
+    if (!maybeStore) {
+        return;
+    }
+    let store = maybeStore as ExternalStore;
+    let doForType = async (type: ItemType) => {
+        let ids: number[] = ((await queryItems(
+            [type],
+            queryNot({
+                a: whichProp[type],
+                leafOp: QueryLeafOp.Like,
+                b: `%${StoreURLIdentifiers[store]}%`,
+            }),
+            undefined,
+            undefined,
+            QueryResponseType.Ids
+        )) as any)[whichElem[type]];
+        ids.map((id: number) => {
+            addTaskCb({
+                itemType: type,
+                itemId: id,
+                integrationId: integration.id,
+                store: store,
+            });
+        })
+    }
+    var promises: Promise<any>[] = [];
+    if (linkSongs) { promises.push(doForType(ItemType.Song)); }
+    if (linkArtists) { promises.push(doForType(ItemType.Artist)); }
+    if (linkAlbums) { promises.push(doForType(ItemType.Album)); }
+    console.log("Awaiting answer...")
+    await Promise.all(promises);
+}
+
+async function doLinking(
+    toLink: { integrationId: number, songs: boolean, artists: boolean, albums: boolean }[],
+    setStatus: any,
+    integrations: IntegrationState[],
+) {
+    console.log("Linking start!", toLink);
+
+    // Start the collecting phase.
+    setStatus({
+        state: BatchJobState.Collecting,
+        numTasks: 0,
+        tasksSuccess: 0,
+        tasksFailed: 0,
+    });
+
+    console.log("Starting collection");
+    var tasks: Task[] = [];
+
+    let collectionPromises = toLink.map((v: any) => {
+        let { integrationId, songs, artists, albums } = v;
+        let integration = integrations.find((i: IntegrationState) => i.id === integrationId);
+        if (!integration) { return; }
+        console.log('integration collect:', integration)
+        return makeTasks(
+            integration,
+            songs,
+            artists,
+            albums,
+            (t: Task) => { tasks.push(t) }
+        );
+    })
+    console.log("Awaiting collection.")
+    await Promise.all(collectionPromises);
+    console.log("Done collecting.", tasks)
+    // Start the linking phase.
+    setStatus((status: BatchJobStatus) => {
+        status.state = BatchJobState.Running;
+        status.numTasks = tasks.length;
+        console.log("Collected status:", status)
+        return status;
+    });
+
+    let makeJob: (t: Task) => Promise<void> = (t: Task) => {
+        let integration = integrations.find((i: IntegrationState) => i.id === t.integrationId);
+        return (async () => {
+            let onSuccess = () => setStatus((s: BatchJobStatus) => { s.tasksSuccess += 1; return s; });
+            let onFail = () => setStatus((s: BatchJobStatus) => { s.tasksFailed += 1; return s; });
+            try {
+                if (integration === undefined) { return; }
+                console.log('integration search:', integration)
+                let _integration = integration as IntegrationState;
+                let searchFuncs: any = {
+                    [ItemType.Song]: (q: any, l: any) => { return _integration.integration.searchSong(q, l) },
+                    [ItemType.Album]: (q: any, l: any) => { return _integration.integration.searchAlbum(q, l) },
+                    [ItemType.Artist]: (q: any, l: any) => { return _integration.integration.searchArtist(q, l) },
+                }
+                // TODO include related items in search
+                let getFuncs: any = {
+                    [ItemType.Song]: getSong,
+                    [ItemType.Album]: getAlbum,
+                    [ItemType.Artist]: getArtist,
+                }
+                let queryFuncs: any = {
+                    [ItemType.Song]: (s: any) => `${s.title}`,
+                    [ItemType.Album]: (s: any) => `${s.name}`,
+                    [ItemType.Artist]: (s: any) => `${s.name}`,
+                }
+                let query = queryFuncs[t.itemType](await getFuncs[t.itemType](t.itemId));
+                let candidates = await searchFuncs[t.itemType](
+                    query,
+                    1,
+                );
+
+                console.log(query, candidates);
+                if (candidates && candidates.length && candidates.length > 0) {
+                    onSuccess();
+                } else {
+                    onFail();
+                }
+            } catch (e) {
+                // Report fail
+                console.log("Error fetching candidates: ", e)
+                onFail();
+            }
+        })();
+    }
+
+    await asyncPool(4, tasks, makeJob);
+
+    // Finalize.
+    setStatus((status: BatchJobStatus) => {
+        status.state = BatchJobState.Idle;
+        console.log("Done running:", status)
+        return status;
+    });
+}
+
+function ProgressDialog(props: {
+    open: boolean,
+    onClose: () => void,
+    status: BatchJobStatus,
+}) {
+    return <Dialog
+        open={props.open}
+        onClose={props.onClose}
+    >
+        <DialogTitle>Batch linking in progress...</DialogTitle>
+        <DialogContent>
+            <DialogContentText>
+                Closing or refreshing this page will interrupt and abort the process.
+            </DialogContentText>
+        </DialogContent>
+    </Dialog>
+}
+
 function ConfirmDialog(props: {
     open: boolean
     onConfirm: () => void,
@@ -42,6 +235,12 @@ export default function BatchLinkDialog(props: {
     let integrations = useIntegrations();
     let classes = useStyles();
     let [confirmDialogOpen, setConfirmDialogOpen] = useState<boolean>(false);
+    let [jobStatus, setJobStatus] = useState<BatchJobStatus>({
+        state: BatchJobState.Idle,
+        numTasks: 0,
+        tasksSuccess: 0,
+        tasksFailed: 0,
+    });
 
     var compatibleIntegrations: Record<ExternalStore, IntegrationState[]> = {
         [ExternalStore.GooglePlayMusic]: [],
@@ -157,7 +356,33 @@ export default function BatchLinkDialog(props: {
         <ConfirmDialog
             open={confirmDialogOpen}
             onClose={() => setConfirmDialogOpen(false)}
-            onConfirm={() => { }}
+            onConfirm={() => {
+                var toLink: any[] = [];
+                Object.keys(storeSettings).forEach((store: string) => {
+                    let s = store as ExternalStore;
+                    let active = Boolean(compatibleIntegrations[s].length);
+
+                    if (active && storeSettings[s].selectedIntegration !== undefined) {
+                        toLink.push({
+                            integrationId: compatibleIntegrations[s][storeSettings[s].selectedIntegration || 0].id,
+                            songs: storeSettings[s].linkSongs,
+                            artists: storeSettings[s].linkArtists,
+                            albums: storeSettings[s].linkAlbums,
+                        });
+                    }
+                });
+                doLinking(
+                    toLink,
+                    setJobStatus,
+                    integrations.state === "Loading" ?
+                        [] : integrations.state,
+                )
+            }}
+        />
+        <ProgressDialog
+            open={jobStatus.state === BatchJobState.Collecting || jobStatus.state === BatchJobState.Running}
+            onClose={() => { }}
+            status={jobStatus}
         />
     </>
 }

client/src/lib/backend/albums.tsx

@@ -0,0 +1,10 @@
+import * as serverApi from '../../api';
+import backendRequest from './request';
+
+export async function getAlbum(id: number) {
+    const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.AlbumDetailsEndpoint.replace(':id', `${id}`))
+    if (!response.ok) {
+        throw new Error("Response to album request not OK: " + JSON.stringify(response));
+    }
+    return await response.json();
+}

client/src/lib/backend/artists.tsx

@@ -0,0 +1,10 @@
+import * as serverApi from '../../api';
+import backendRequest from './request';
+
+export async function getArtist(id: number) {
+    const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.ArtistDetailsEndpoint.replace(':id', `${id}`))
+    if (!response.ok) {
+        throw new Error("Response to artist request not OK: " + JSON.stringify(response));
+    }
+    return await response.json();
+}

client/src/lib/backend/songs.tsx

@@ -0,0 +1,10 @@
+import * as serverApi from '../../api';
+import backendRequest from './request';
+
+export async function getSong(id: number) {
+    const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.SongDetailsEndpoint.replace(':id', `${id}`))
+    if (!response.ok) {
+        throw new Error("Response to song request not OK: " + JSON.stringify(response));
+    }
+    return await response.json();
+}

client/src/lib/query/Query.tsx

@@ -42,6 +42,7 @@ export function isLeafElem(q: QueryElem): q is QueryLeafElem {
 export enum QueryNodeOp {
     And = 0,
     Or,
+    Not,
 }
 
 export interface QueryNodeElem {
@@ -67,6 +68,13 @@ export function queryAnd(...args: QueryElem[]) {
     };
 }
 
+export function queryNot(arg: QueryElem) {
+    return {
+        operands: [arg],
+        nodeOp: QueryNodeOp.Not,
+    }
+}
+
 export type QueryElem = QueryLeafElem | QueryNodeElem;
 
 // Take a query and add placeholders. The placeholders are empty
@@ -76,7 +84,7 @@ export type QueryElem = QueryLeafElem | QueryNodeElem;
 // placeholders for all AND/OR combinations with existing nodes.
 export function addPlaceholders(
     q: QueryElem | null,
-    inNode: null | QueryNodeOp.And | QueryNodeOp.Or,
+    inNode: null | QueryNodeOp,
 ): QueryElem {
 
     const makePlaceholder = () => {
@@ -90,6 +98,7 @@ export function addPlaceholders(
     const otherOp: Record<QueryNodeOp, QueryNodeOp> = {
         [QueryNodeOp.And]: QueryNodeOp.Or,
         [QueryNodeOp.Or]: QueryNodeOp.And,
+        [QueryNodeOp.Not]: QueryNodeOp.Not, // TODO fix this
     }
 
     if (q == null) {
@@ -188,6 +197,7 @@ export function toApiQuery(q: QueryElem) : serverApi.Query {
     const nodeOpsMapping: any = {
         [QueryNodeOp.And]: serverApi.QueryElemOp.And,
         [QueryNodeOp.Or]: serverApi.QueryElemOp.Or,
+        [QueryNodeOp.Not]: serverApi.QueryElemOp.Not,
     }
     
     if(isLeafElem(q) && isTagQueryInfo(q.b)) {

server/endpoints/Query.ts

@@ -92,24 +92,44 @@ enum WhereType {
     Or,
 };
 
-function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
+function getSQLValue(val: any) {
+    console.log("Value:", val)
+    if (typeof val === 'string') {
+        return `'${val}'`;
+    } else if (typeof val === 'number') {
+        return `${val}`;
+    }
+    throw new Error("unimplemented SQL value type.");
+}
+
+function getSQLValues(vals: any[]) {
+    if (vals.length === 0) { return '()' }
+    let r = `(${getSQLValue(vals[0])}`;
+    for (let i: number = 1; i < vals.length; i++) {
+        r += `, ${getSQLValue(vals[i])}`;
+    }
+    r += ')';
+    return r;
+}
+
+function getLeafWhere(queryElem: api.QueryElem): string {
     const simpleLeafOps: Record<any, string> = {
         [api.QueryFilterOp.Eq]: "=",
         [api.QueryFilterOp.Ne]: "!=",
-        [api.QueryFilterOp.Like]: "like",
+        [api.QueryFilterOp.Like]: "LIKE",
     }
 
     const propertyKeys = {
-        [api.QueryElemProperty.songTitle]: 'songs.title',
-        [api.QueryElemProperty.songId]: 'songs.id',
-        [api.QueryElemProperty.artistName]: 'artists.name',
-        [api.QueryElemProperty.artistId]: 'artists.id',
-        [api.QueryElemProperty.albumName]: 'albums.name',
-        [api.QueryElemProperty.albumId]: 'albums.id',
-        [api.QueryElemProperty.tagId]: 'tags.id',
-        [api.QueryElemProperty.songStoreLinks]: 'songs.storeLinks',
-        [api.QueryElemProperty.artistStoreLinks]: 'artists.storeLinks',
-        [api.QueryElemProperty.albumStoreLinks]: 'albums.storeLinks',
+        [api.QueryElemProperty.songTitle]: '`songs`.`title`',
+        [api.QueryElemProperty.songId]: '`songs`.`id`',
+        [api.QueryElemProperty.artistName]: '`artists`.`name`',
+        [api.QueryElemProperty.artistId]: '`artists`.`id`',
+        [api.QueryElemProperty.albumName]: '`albums`.`name`',
+        [api.QueryElemProperty.albumId]: '`albums`.`id`',
+        [api.QueryElemProperty.tagId]: '`tags`.`id`',
+        [api.QueryElemProperty.songStoreLinks]: '`songs`.`storeLinks`',
+        [api.QueryElemProperty.artistStoreLinks]: '`artists`.`storeLinks`',
+        [api.QueryElemProperty.albumStoreLinks]: '`albums`.`storeLinks`',
     }
 
     if (!queryElem.propOperator) throw "Cannot create where clause without an operator.";
@@ -120,61 +140,49 @@ function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType)
         : (queryElem.propOperand || "");
 
     if (Object.keys(simpleLeafOps).includes(operator)) {
-        if (type == WhereType.And) {
-            return knexQuery.andWhere(a, simpleLeafOps[operator], b);
-        } else if (type == WhereType.Or) {
-            return knexQuery.orWhere(a, simpleLeafOps[operator], b);
-        }
+        return `(${a} ${simpleLeafOps[operator]} ${getSQLValue(b)})`;
     } else if (operator == api.QueryFilterOp.In) {
-        if (type == WhereType.And) {
-            return knexQuery.whereIn(a, b);
-        } else if (type == WhereType.Or) {
-            return knexQuery.orWhereIn(a, b);
-        }
+        return `(${a} IN ${getSQLValues(b)})`
     } else if (operator == api.QueryFilterOp.NotIn) {
-        if (type == WhereType.And) {
-            return knexQuery.whereNotIn(a, b);
-        } else if (type == WhereType.Or) {
-            return knexQuery.orWhereNotIn(a, b);
-        }
+        return `(${a} NOT IN ${getSQLValues(b)})`
     }
 
     throw "Query filter not implemented";
 }
 
-function addBranchWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
-    if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.And) {
-        var q = knexQuery;
-        queryElem.children.forEach((child: api.QueryElem) => {
-            q = addWhere(q, child, type);
-        })
-        return q;
-    } else if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.Or) {
-        var q = knexQuery;
-        const c = queryElem.children;
-        const f = function (this: any) {
-            for (var i = 0; i < c.length; i++) {
-                addWhere(this, c[i], WhereType.Or);
-            }
-        }
-        if (type == WhereType.And) {
-            return q.where(f);
-        } else if (type == WhereType.Or) {
-            return q.orWhere(f);
+function getNodeWhere(queryElem: api.QueryElem): string {
+    let ops = {
+        [api.QueryElemOp.And]: 'AND',
+        [api.QueryElemOp.Or]: 'OR',
+        [api.QueryElemOp.Not]: 'NOT',
+    }
+    let buildList = (subqueries: api.QueryElem[], operator: api.QueryElemOp) => {
+        if (subqueries.length === 0) { return 'true' }
+        let r = `(${getWhere(subqueries[0])}`;
+        for (let i: number = 1; i < subqueries.length; i++) {
+            r += ` ${ops[operator]} ${getWhere(subqueries[i])}`;
         }
+        r += ')';
+        return r;
     }
-}
 
-function addWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
-    if (queryElem.prop) {
-        // Leaf node.
-        return addLeafWhere(knexQuery, queryElem, type);
-    } else if (queryElem.children) {
-        // Branch node.
-        return addBranchWhere(knexQuery, queryElem, type);
+    if (queryElem.children && queryElem.childrenOperator && queryElem.children.length) {
+        if (queryElem.childrenOperator === api.QueryElemOp.And ||
+            queryElem.childrenOperator === api.QueryElemOp.Or) {
+            return buildList(queryElem.children, queryElem.childrenOperator)
+        } else if (queryElem.childrenOperator === api.QueryElemOp.Not &&
+            queryElem.children.length === 1) {
+            return `NOT ${getWhere(queryElem.children[0])}`
+        }
     }
 
-    return knexQuery;
+    throw new Error('invalid query')
+}
+
+function getWhere(queryElem: api.QueryElem): string {
+    if (queryElem.prop) { return getLeafWhere(queryElem); }
+    if (queryElem.children) { return getNodeWhere(queryElem); }
+    return "true";
 }
 
 const objectColumns = {
@@ -210,7 +218,7 @@ function constructQuery(knex: Knex, userId: number, queryFor: ObjectType, queryE
     })
 
     // Apply filtering.
-    q = addWhere(q, queryElem, WhereType.And);
+    q = q.andWhereRaw(getWhere(queryElem));
 
     // Apply ordering
     const orderKeys = {

server/test/integration/flows/QueryFlow.js

@@ -197,6 +197,7 @@ describe('POST /query with several songs and filters', () => {
         }
 
         async function checkArtistIdIn(req) {
+            console.log("HERE!")
             await req
                 .post('/query')
                 .send({