diff --git a/client/package-lock.json b/client/package-lock.json index 3718ad2..0e5418c 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -2066,6 +2066,11 @@ } } }, + "@types/tiny-async-pool": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/tiny-async-pool/-/tiny-async-pool-1.0.0.tgz", + "integrity": "sha512-d8RK1jg/piCgv5/jD8ta8uJOE10tU8MWExzL1Kf1kOjMaTuL5cW0eZ9ax001SSYa4Ecg6xzZBh/jM4GB7+5OAg==" + }, "@types/uuid": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", @@ -13285,6 +13290,22 @@ "resolved": "https://registry.npmjs.org/timsort/-/timsort-0.3.0.tgz", "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=" }, + "tiny-async-pool": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.2.0.tgz", + "integrity": "sha512-PY/OiSenYGBU3c1nTuP1HLKRkhKFDXsAibYI5GeHbHw2WVpt6OFzAPIRP94dGnS66Jhrkheim2CHAXUNI4XwMg==", + "requires": { + "semver": "^5.5.0", + "yaassertion": "^1.0.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, "tiny-invariant": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.1.0.tgz", @@ -14585,6 +14606,11 @@ "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==" }, + "yaassertion": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz", + "integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg==" + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/client/package.json b/client/package.json index 444fa2b..28f7ee6 100644 --- a/client/package.json +++ b/client/package.json @@ -15,6 +15,7 @@ "@types/react-dom": "^16.9.0", "@types/react-router": "^5.1.8", "@types/react-router-dom": "^5.1.5", + "@types/tiny-async-pool": "^1.0.0", "@types/uuid": "^8.3.0", "jsurl": "^0.1.5", "lodash": "^4.17.20", @@ -27,6 +28,7 @@ "react-error-boundary": "^3.0.2", "react-router-dom": "^5.2.0", "react-scripts": "^3.4.3", + "tiny-async-pool": "^1.2.0", "ts-enum-util": "^4.0.2", "typescript": "~3.7.2", "uuid": "^8.3.0" diff --git a/client/src/api.ts b/client/src/api.ts index 898878f..1de23c8 100644 --- a/client/src/api.ts +++ b/client/src/api.ts @@ -67,6 +67,7 @@ export const QueryEndpoint = '/query'; export enum QueryElemOp { And = "AND", Or = "OR", + Not = "NOT", } export enum QueryFilterOp { Eq = "EQ", diff --git a/client/src/components/windows/manage_links/BatchLinkDialog.tsx b/client/src/components/windows/manage_links/BatchLinkDialog.tsx index c357b7e..e955882 100644 --- a/client/src/components/windows/manage_links/BatchLinkDialog.tsx +++ b/client/src/components/windows/manage_links/BatchLinkDialog.tsx @@ -1,9 +1,16 @@ -import React, { useState } from 'react'; +import React, { Dispatch, SetStateAction, useEffect, useState } from 'react'; import { Box, Button, Checkbox, createStyles, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, Divider, FormControlLabel, List, ListItem, ListItemIcon, ListItemText, makeStyles, MenuItem, Paper, Select, Theme, Typography } from "@material-ui/core"; import StoreLinkIcon from '../../common/StoreLinkIcon'; import { $enum } from 'ts-enum-util'; import { IntegrationState, useIntegrations } from '../../../lib/integration/useIntegrations'; -import { ExternalStore, IntegrationStores, IntegrationType } from '../../../api'; +import { ExternalStore, IntegrationStores, IntegrationType, ItemType, QueryResponseType, StoreURLIdentifiers } from '../../../api'; +import { start } from 'repl'; +import { QueryLeafBy, QueryLeafOp, QueryNodeOp, queryNot } from '../../../lib/query/Query'; +import { queryAlbums, queryArtists, queryItems, querySongs } from '../../../lib/backend/queries'; +import asyncPool from "tiny-async-pool"; +import { getSong } from '../../../lib/backend/songs'; +import { getAlbum } from '../../../lib/backend/albums'; +import { getArtist } from '../../../lib/backend/artists'; const useStyles = makeStyles((theme: Theme) => createStyles({ @@ -13,6 +20,192 @@ const useStyles = makeStyles((theme: Theme) => }) ); +enum BatchJobState { + Idle = 0, + Collecting, + Running, +} + +interface Task { + itemType: ItemType, + itemId: number, + integrationId: number, + store: ExternalStore, +} + +interface BatchJobStatus { + state: BatchJobState, + numTasks: number, + tasksSuccess: number, + tasksFailed: number, +} + +async function makeTasks( + integration: IntegrationState, + linkSongs: boolean, + linkArtists: boolean, + linkAlbums: boolean, + addTaskCb: (t: Task) => void, +) { + let whichProp: any = { + [ItemType.Song]: QueryLeafBy.SongStoreLinks, + [ItemType.Artist]: QueryLeafBy.ArtistStoreLinks, + [ItemType.Album]: QueryLeafBy.AlbumStoreLinks, + } + let whichElem: any = { + [ItemType.Song]: 'songs', + [ItemType.Artist]: 'artists', + [ItemType.Album]: 'albums', + } + let maybeStore = integration.integration.providesStoreLink(); + if (!maybeStore) { + return; + } + let store = maybeStore as ExternalStore; + let doForType = async (type: ItemType) => { + let ids: number[] = ((await queryItems( + [type], + queryNot({ + a: whichProp[type], + leafOp: QueryLeafOp.Like, + b: `%${StoreURLIdentifiers[store]}%`, + }), + undefined, + undefined, + QueryResponseType.Ids + )) as any)[whichElem[type]]; + ids.map((id: number) => { + addTaskCb({ + itemType: type, + itemId: id, + integrationId: integration.id, + store: store, + }); + }) + } + var promises: Promise[] = []; + if (linkSongs) { promises.push(doForType(ItemType.Song)); } + if (linkArtists) { promises.push(doForType(ItemType.Artist)); } + if (linkAlbums) { promises.push(doForType(ItemType.Album)); } + console.log("Awaiting answer...") + await Promise.all(promises); +} + +async function doLinking( + toLink: { integrationId: number, songs: boolean, artists: boolean, albums: boolean }[], + setStatus: any, + integrations: IntegrationState[], +) { + console.log("Linking start!", toLink); + + // Start the collecting phase. + setStatus({ + state: BatchJobState.Collecting, + numTasks: 0, + tasksSuccess: 0, + tasksFailed: 0, + }); + + console.log("Starting collection"); + var tasks: Task[] = []; + + let collectionPromises = toLink.map((v: any) => { + let { integrationId, songs, artists, albums } = v; + let integration = integrations.find((i: IntegrationState) => i.id === integrationId); + if (!integration) { return; } + console.log('integration collect:', integration) + return makeTasks( + integration, + songs, + artists, + albums, + (t: Task) => { tasks.push(t) } + ); + }) + console.log("Awaiting collection.") + await Promise.all(collectionPromises); + console.log("Done collecting.", tasks) + // Start the linking phase. + setStatus((status: BatchJobStatus) => { + status.state = BatchJobState.Running; + status.numTasks = tasks.length; + console.log("Collected status:", status) + return status; + }); + + let makeJob: (t: Task) => Promise = (t: Task) => { + let integration = integrations.find((i: IntegrationState) => i.id === t.integrationId); + return (async () => { + let onSuccess = () => setStatus((s: BatchJobStatus) => { s.tasksSuccess += 1; return s; }); + let onFail = () => setStatus((s: BatchJobStatus) => { s.tasksFailed += 1; return s; }); + try { + if (integration === undefined) { return; } + console.log('integration search:', integration) + let _integration = integration as IntegrationState; + let searchFuncs: any = { + [ItemType.Song]: (q: any, l: any) => { return _integration.integration.searchSong(q, l) }, + [ItemType.Album]: (q: any, l: any) => { return _integration.integration.searchAlbum(q, l) }, + [ItemType.Artist]: (q: any, l: any) => { return _integration.integration.searchArtist(q, l) }, + } + // TODO include related items in search + let getFuncs: any = { + [ItemType.Song]: getSong, + [ItemType.Album]: getAlbum, + [ItemType.Artist]: getArtist, + } + let queryFuncs: any = { + [ItemType.Song]: (s: any) => `${s.title}`, + [ItemType.Album]: (s: any) => `${s.name}`, + [ItemType.Artist]: (s: any) => `${s.name}`, + } + let query = queryFuncs[t.itemType](await getFuncs[t.itemType](t.itemId)); + let candidates = await searchFuncs[t.itemType]( + query, + 1, + ); + + console.log(query, candidates); + if (candidates && candidates.length && candidates.length > 0) { + onSuccess(); + } else { + onFail(); + } + } catch (e) { + // Report fail + console.log("Error fetching candidates: ", e) + onFail(); + } + })(); + } + + await asyncPool(4, tasks, makeJob); + + // Finalize. + setStatus((status: BatchJobStatus) => { + status.state = BatchJobState.Idle; + console.log("Done running:", status) + return status; + }); +} + +function ProgressDialog(props: { + open: boolean, + onClose: () => void, + status: BatchJobStatus, +}) { + return + Batch linking in progress... + + + Closing or refreshing this page will interrupt and abort the process. + + + +} + function ConfirmDialog(props: { open: boolean onConfirm: () => void, @@ -42,6 +235,12 @@ export default function BatchLinkDialog(props: { let integrations = useIntegrations(); let classes = useStyles(); let [confirmDialogOpen, setConfirmDialogOpen] = useState(false); + let [jobStatus, setJobStatus] = useState({ + state: BatchJobState.Idle, + numTasks: 0, + tasksSuccess: 0, + tasksFailed: 0, + }); var compatibleIntegrations: Record = { [ExternalStore.GooglePlayMusic]: [], @@ -157,7 +356,33 @@ export default function BatchLinkDialog(props: { setConfirmDialogOpen(false)} - onConfirm={() => { }} + onConfirm={() => { + var toLink: any[] = []; + Object.keys(storeSettings).forEach((store: string) => { + let s = store as ExternalStore; + let active = Boolean(compatibleIntegrations[s].length); + + if (active && storeSettings[s].selectedIntegration !== undefined) { + toLink.push({ + integrationId: compatibleIntegrations[s][storeSettings[s].selectedIntegration || 0].id, + songs: storeSettings[s].linkSongs, + artists: storeSettings[s].linkArtists, + albums: storeSettings[s].linkAlbums, + }); + } + }); + doLinking( + toLink, + setJobStatus, + integrations.state === "Loading" ? + [] : integrations.state, + ) + }} + /> + { }} + status={jobStatus} /> } \ No newline at end of file diff --git a/client/src/lib/backend/albums.tsx b/client/src/lib/backend/albums.tsx new file mode 100644 index 0000000..ce983be --- /dev/null +++ b/client/src/lib/backend/albums.tsx @@ -0,0 +1,10 @@ +import * as serverApi from '../../api'; +import backendRequest from './request'; + +export async function getAlbum(id: number) { + const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.AlbumDetailsEndpoint.replace(':id', `${id}`)) + if (!response.ok) { + throw new Error("Response to album request not OK: " + JSON.stringify(response)); + } + return await response.json(); +} diff --git a/client/src/lib/backend/artists.tsx b/client/src/lib/backend/artists.tsx new file mode 100644 index 0000000..097036c --- /dev/null +++ b/client/src/lib/backend/artists.tsx @@ -0,0 +1,10 @@ +import * as serverApi from '../../api'; +import backendRequest from './request'; + +export async function getArtist(id: number) { + const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.ArtistDetailsEndpoint.replace(':id', `${id}`)) + if (!response.ok) { + throw new Error("Response to artist request not OK: " + JSON.stringify(response)); + } + return await response.json(); +} diff --git a/client/src/lib/backend/songs.tsx b/client/src/lib/backend/songs.tsx new file mode 100644 index 0000000..6fc8f0c --- /dev/null +++ b/client/src/lib/backend/songs.tsx @@ -0,0 +1,10 @@ +import * as serverApi from '../../api'; +import backendRequest from './request'; + +export async function getSong(id: number) { + const response = await backendRequest((process.env.REACT_APP_BACKEND || "") + serverApi.SongDetailsEndpoint.replace(':id', `${id}`)) + if (!response.ok) { + throw new Error("Response to song request not OK: " + JSON.stringify(response)); + } + return await response.json(); +} diff --git a/client/src/lib/query/Query.tsx b/client/src/lib/query/Query.tsx index dcd96c7..eda2cdb 100644 --- a/client/src/lib/query/Query.tsx +++ b/client/src/lib/query/Query.tsx @@ -42,6 +42,7 @@ export function isLeafElem(q: QueryElem): q is QueryLeafElem { export enum QueryNodeOp { And = 0, Or, + Not, } export interface QueryNodeElem { @@ -67,6 +68,13 @@ export function queryAnd(...args: QueryElem[]) { }; } +export function queryNot(arg: QueryElem) { + return { + operands: [arg], + nodeOp: QueryNodeOp.Not, + } +} + export type QueryElem = QueryLeafElem | QueryNodeElem; // Take a query and add placeholders. The placeholders are empty @@ -76,7 +84,7 @@ export type QueryElem = QueryLeafElem | QueryNodeElem; // placeholders for all AND/OR combinations with existing nodes. export function addPlaceholders( q: QueryElem | null, - inNode: null | QueryNodeOp.And | QueryNodeOp.Or, + inNode: null | QueryNodeOp, ): QueryElem { const makePlaceholder = () => { @@ -90,6 +98,7 @@ export function addPlaceholders( const otherOp: Record = { [QueryNodeOp.And]: QueryNodeOp.Or, [QueryNodeOp.Or]: QueryNodeOp.And, + [QueryNodeOp.Not]: QueryNodeOp.Not, // TODO fix this } if (q == null) { @@ -188,6 +197,7 @@ export function toApiQuery(q: QueryElem) : serverApi.Query { const nodeOpsMapping: any = { [QueryNodeOp.And]: serverApi.QueryElemOp.And, [QueryNodeOp.Or]: serverApi.QueryElemOp.Or, + [QueryNodeOp.Not]: serverApi.QueryElemOp.Not, } if(isLeafElem(q) && isTagQueryInfo(q.b)) { diff --git a/server/endpoints/Query.ts b/server/endpoints/Query.ts index 13caa2d..ea5321a 100644 --- a/server/endpoints/Query.ts +++ b/server/endpoints/Query.ts @@ -92,24 +92,44 @@ enum WhereType { Or, }; -function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) { +function getSQLValue(val: any) { + console.log("Value:", val) + if (typeof val === 'string') { + return `'${val}'`; + } else if (typeof val === 'number') { + return `${val}`; + } + throw new Error("unimplemented SQL value type."); +} + +function getSQLValues(vals: any[]) { + if (vals.length === 0) { return '()' } + let r = `(${getSQLValue(vals[0])}`; + for (let i: number = 1; i < vals.length; i++) { + r += `, ${getSQLValue(vals[i])}`; + } + r += ')'; + return r; +} + +function getLeafWhere(queryElem: api.QueryElem): string { const simpleLeafOps: Record = { [api.QueryFilterOp.Eq]: "=", [api.QueryFilterOp.Ne]: "!=", - [api.QueryFilterOp.Like]: "like", + [api.QueryFilterOp.Like]: "LIKE", } const propertyKeys = { - [api.QueryElemProperty.songTitle]: 'songs.title', - [api.QueryElemProperty.songId]: 'songs.id', - [api.QueryElemProperty.artistName]: 'artists.name', - [api.QueryElemProperty.artistId]: 'artists.id', - [api.QueryElemProperty.albumName]: 'albums.name', - [api.QueryElemProperty.albumId]: 'albums.id', - [api.QueryElemProperty.tagId]: 'tags.id', - [api.QueryElemProperty.songStoreLinks]: 'songs.storeLinks', - [api.QueryElemProperty.artistStoreLinks]: 'artists.storeLinks', - [api.QueryElemProperty.albumStoreLinks]: 'albums.storeLinks', + [api.QueryElemProperty.songTitle]: '`songs`.`title`', + [api.QueryElemProperty.songId]: '`songs`.`id`', + [api.QueryElemProperty.artistName]: '`artists`.`name`', + [api.QueryElemProperty.artistId]: '`artists`.`id`', + [api.QueryElemProperty.albumName]: '`albums`.`name`', + [api.QueryElemProperty.albumId]: '`albums`.`id`', + [api.QueryElemProperty.tagId]: '`tags`.`id`', + [api.QueryElemProperty.songStoreLinks]: '`songs`.`storeLinks`', + [api.QueryElemProperty.artistStoreLinks]: '`artists`.`storeLinks`', + [api.QueryElemProperty.albumStoreLinks]: '`albums`.`storeLinks`', } if (!queryElem.propOperator) throw "Cannot create where clause without an operator."; @@ -120,61 +140,49 @@ function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) : (queryElem.propOperand || ""); if (Object.keys(simpleLeafOps).includes(operator)) { - if (type == WhereType.And) { - return knexQuery.andWhere(a, simpleLeafOps[operator], b); - } else if (type == WhereType.Or) { - return knexQuery.orWhere(a, simpleLeafOps[operator], b); - } + return `(${a} ${simpleLeafOps[operator]} ${getSQLValue(b)})`; } else if (operator == api.QueryFilterOp.In) { - if (type == WhereType.And) { - return knexQuery.whereIn(a, b); - } else if (type == WhereType.Or) { - return knexQuery.orWhereIn(a, b); - } + return `(${a} IN ${getSQLValues(b)})` } else if (operator == api.QueryFilterOp.NotIn) { - if (type == WhereType.And) { - return knexQuery.whereNotIn(a, b); - } else if (type == WhereType.Or) { - return knexQuery.orWhereNotIn(a, b); - } + return `(${a} NOT IN ${getSQLValues(b)})` } throw "Query filter not implemented"; } -function addBranchWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) { - if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.And) { - var q = knexQuery; - queryElem.children.forEach((child: api.QueryElem) => { - q = addWhere(q, child, type); - }) - return q; - } else if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.Or) { - var q = knexQuery; - const c = queryElem.children; - const f = function (this: any) { - for (var i = 0; i < c.length; i++) { - addWhere(this, c[i], WhereType.Or); - } - } - if (type == WhereType.And) { - return q.where(f); - } else if (type == WhereType.Or) { - return q.orWhere(f); +function getNodeWhere(queryElem: api.QueryElem): string { + let ops = { + [api.QueryElemOp.And]: 'AND', + [api.QueryElemOp.Or]: 'OR', + [api.QueryElemOp.Not]: 'NOT', + } + let buildList = (subqueries: api.QueryElem[], operator: api.QueryElemOp) => { + if (subqueries.length === 0) { return 'true' } + let r = `(${getWhere(subqueries[0])}`; + for (let i: number = 1; i < subqueries.length; i++) { + r += ` ${ops[operator]} ${getWhere(subqueries[i])}`; } + r += ')'; + return r; } -} -function addWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) { - if (queryElem.prop) { - // Leaf node. - return addLeafWhere(knexQuery, queryElem, type); - } else if (queryElem.children) { - // Branch node. - return addBranchWhere(knexQuery, queryElem, type); + if (queryElem.children && queryElem.childrenOperator && queryElem.children.length) { + if (queryElem.childrenOperator === api.QueryElemOp.And || + queryElem.childrenOperator === api.QueryElemOp.Or) { + return buildList(queryElem.children, queryElem.childrenOperator) + } else if (queryElem.childrenOperator === api.QueryElemOp.Not && + queryElem.children.length === 1) { + return `NOT ${getWhere(queryElem.children[0])}` + } } - return knexQuery; + throw new Error('invalid query') +} + +function getWhere(queryElem: api.QueryElem): string { + if (queryElem.prop) { return getLeafWhere(queryElem); } + if (queryElem.children) { return getNodeWhere(queryElem); } + return "true"; } const objectColumns = { @@ -210,7 +218,7 @@ function constructQuery(knex: Knex, userId: number, queryFor: ObjectType, queryE }) // Apply filtering. - q = addWhere(q, queryElem, WhereType.And); + q = q.andWhereRaw(getWhere(queryElem)); // Apply ordering const orderKeys = { diff --git a/server/test/integration/flows/QueryFlow.js b/server/test/integration/flows/QueryFlow.js index 9da8da6..3e95dd5 100644 --- a/server/test/integration/flows/QueryFlow.js +++ b/server/test/integration/flows/QueryFlow.js @@ -197,6 +197,7 @@ describe('POST /query with several songs and filters', () => { } async function checkArtistIdIn(req) { + console.log("HERE!") await req .post('/query') .send({