import * as api from '../../client/src/api/api'; import { EndpointError, EndpointHandler, handleErrorsInEndpoint } from '../endpoints/types'; import Knex from 'knex'; import asJson from '../lib/asJson'; export function toApiTag(dbObj: any): api.Tag { return { mbApi_typename: "tag", tagId: dbObj['tags.id'], name: dbObj['tags.name'], parentId: dbObj['tags.parentId'], parent: dbObj.parent ? toApiTag(dbObj.parent) : undefined, }; } export function toApiArtist(dbObj: any): api.Artist { return { mbApi_typename: "artist", artistId: dbObj['artists.id'], name: dbObj['artists.name'], storeLinks: asJson(dbObj['artists.storeLinks']), }; } export function toApiTrack(dbObj: any, artists: any[], tags: any[], albums: any[]): api.Track { return { mbApi_typename: "track", trackId: dbObj['tracks.id'], name: dbObj['tracks.name'], storeLinks: asJson(dbObj['tracks.storeLinks']), artists: artists.map((artist: any) => { return toApiArtist(artist); }), tags: tags.map((tag: any) => { return toApiTag(tag); }), album: albums.length > 0 ? toApiAlbum(albums[0]) : null, } } export function toApiAlbum(dbObj: any): api.Album { return { mbApi_typename: "album", albumId: dbObj['albums.id'], name: dbObj['albums.name'], storeLinks: asJson(dbObj['albums.storeLinks']), }; } enum ObjectType { Track = 0, Artist, Tag, Album, } // To keep track of which database objects are needed to filter on // certain properties. const propertyObjects: Record = { [api.QueryElemProperty.albumId]: ObjectType.Album, [api.QueryElemProperty.albumName]: ObjectType.Album, [api.QueryElemProperty.artistId]: ObjectType.Artist, [api.QueryElemProperty.artistName]: ObjectType.Artist, [api.QueryElemProperty.trackId]: ObjectType.Track, [api.QueryElemProperty.trackName]: ObjectType.Track, [api.QueryElemProperty.tagId]: ObjectType.Tag, [api.QueryElemProperty.tagName]: ObjectType.Tag, [api.QueryElemProperty.trackStoreLinks]: ObjectType.Track, [api.QueryElemProperty.artistStoreLinks]: ObjectType.Artist, [api.QueryElemProperty.albumStoreLinks]: ObjectType.Album, } // To keep track of the tables in which objects are stored. const objectTables: Record = { [ObjectType.Album]: 'albums', [ObjectType.Artist]: 'artists', [ObjectType.Track]: 'tracks', [ObjectType.Tag]: 'tags', } // To keep track of linking tables between objects. const linkingTables: any = [ [[ObjectType.Track, ObjectType.Artist], 'tracks_artists'], [[ObjectType.Track, ObjectType.Tag], 'tracks_tags'], [[ObjectType.Artist, ObjectType.Album], 'artists_albums'], [[ObjectType.Artist, ObjectType.Tag], 'artists_tags'], [[ObjectType.Album, ObjectType.Tag], 'albums_tags'], ] function getLinkingTable(a: ObjectType, b: ObjectType): string | undefined { var res: string | undefined = undefined; linkingTables.forEach((row: any) => { if (row[0].includes(a) && row[0].includes(b)) { res = row[1]; } }) return res; } // To keep track of linking columns between objects. const linkingColumns: any = [ [[ObjectType.Track, ObjectType.Album], 'tracks.album'], ] function getLinkingColumn(a: ObjectType, b: ObjectType): string | undefined { var res: string | undefined = undefined; linkingColumns.forEach((row: any) => { if (row[0].includes(a) && row[0].includes(b)) { res = row[1]; } }) return res; } // To keep track of ID fields used in linking tables. const linkingTableIdNames: Record = { [ObjectType.Album]: 'albumId', [ObjectType.Artist]: 'artistId', [ObjectType.Track]: 'trackId', [ObjectType.Tag]: 'tagId', } function getRequiredDatabaseObjects(queryElem: api.QueryElem): Set { if (queryElem.prop) { // Leaf node. return new Set([propertyObjects[queryElem.prop]]); } else if (queryElem.children) { // Branch node. var r = new Set(); queryElem.children.forEach((child: api.QueryElem) => { getRequiredDatabaseObjects(child).forEach(object => r.add(object)); }); return r; } return new Set([]); } function addJoin(knexQuery: any, base: ObjectType, other: ObjectType) { const linkTable = getLinkingTable(base, other); const linkColumn = getLinkingColumn(base, other); const baseTable = objectTables[base]; const otherTable = objectTables[other]; if (linkTable) { return knexQuery .join(linkTable, { [baseTable + '.id']: linkTable + '.' + linkingTableIdNames[base] }) .join(otherTable, { [otherTable + '.id']: linkTable + '.' + linkingTableIdNames[other] }); } else if (linkColumn) { return knexQuery .join(otherTable, { [linkColumn]: otherTable + '.id' }); } } enum WhereType { And = 0, Or, }; function getSQLValue(val: any) { console.log("Value:", val) if (typeof val === 'string') { return `'${val}'`; } else if (typeof val === 'number') { return `${val}`; } throw new Error("unimplemented SQL value type."); } function getSQLValues(vals: any[]) { if (vals.length === 0) { return '()' } let r = `(${getSQLValue(vals[0])}`; for (let i: number = 1; i < vals.length; i++) { r += `, ${getSQLValue(vals[i])}`; } r += ')'; return r; } function getLeafWhere(queryElem: api.QueryElem): string { const simpleLeafOps: Record = { [api.QueryLeafOp.Eq]: "=", [api.QueryLeafOp.Ne]: "!=", [api.QueryLeafOp.Like]: "LIKE", } const propertyKeys = { [api.QueryElemProperty.trackName]: '`tracks`.`name`', [api.QueryElemProperty.trackId]: '`tracks`.`id`', [api.QueryElemProperty.artistName]: '`artists`.`name`', [api.QueryElemProperty.artistId]: '`artists`.`id`', [api.QueryElemProperty.albumName]: '`albums`.`name`', [api.QueryElemProperty.albumId]: '`albums`.`id`', [api.QueryElemProperty.tagId]: '`tags`.`id`', [api.QueryElemProperty.tagName]: '`tags`.`name`', [api.QueryElemProperty.trackStoreLinks]: '`tracks`.`storeLinks`', [api.QueryElemProperty.artistStoreLinks]: '`artists`.`storeLinks`', [api.QueryElemProperty.albumStoreLinks]: '`albums`.`storeLinks`', } if (!queryElem.propOperator) throw "Cannot create where clause without an operator."; const operator = queryElem.propOperator || api.QueryLeafOp.Eq; const a = queryElem.prop && propertyKeys[queryElem.prop]; const b = operator === api.QueryLeafOp.Like ? '%' + (queryElem.propOperand || "") + '%' : (queryElem.propOperand || ""); if (Object.keys(simpleLeafOps).includes(operator)) { return `(${a} ${simpleLeafOps[operator]} ${getSQLValue(b)})`; } else if (operator == api.QueryLeafOp.In) { return `(${a} IN ${getSQLValues(b)})` } else if (operator == api.QueryLeafOp.NotIn) { return `(${a} NOT IN ${getSQLValues(b)})` } throw "Query filter not implemented"; } function getNodeWhere(queryElem: api.QueryElem): string { let ops = { [api.QueryNodeOp.And]: 'AND', [api.QueryNodeOp.Or]: 'OR', [api.QueryNodeOp.Not]: 'NOT', } let buildList = (subqueries: api.QueryElem[], operator: api.QueryNodeOp) => { if (subqueries.length === 0) { return 'true' } let r = `(${getWhere(subqueries[0])}`; for (let i: number = 1; i < subqueries.length; i++) { r += ` ${ops[operator]} ${getWhere(subqueries[i])}`; } r += ')'; return r; } if (queryElem.children && queryElem.childrenOperator && queryElem.children.length) { if (queryElem.childrenOperator === api.QueryNodeOp.And || queryElem.childrenOperator === api.QueryNodeOp.Or) { return buildList(queryElem.children, queryElem.childrenOperator) } else if (queryElem.childrenOperator === api.QueryNodeOp.Not && queryElem.children.length === 1) { return `NOT ${getWhere(queryElem.children[0])}` } } throw new Error('invalid query') } function getWhere(queryElem: api.QueryElem): string { if (queryElem.prop) { return getLeafWhere(queryElem); } if (queryElem.children) { return getNodeWhere(queryElem); } return "true"; } const objectColumns = { [ObjectType.Track]: ['tracks.id as tracks.id', 'tracks.name as tracks.name', 'tracks.storeLinks as tracks.storeLinks', 'tracks.album as tracks.album'], [ObjectType.Artist]: ['artists.id as artists.id', 'artists.name as artists.name', 'artists.storeLinks as artists.storeLinks'], [ObjectType.Album]: ['albums.id as albums.id', 'albums.name as albums.name', 'albums.storeLinks as albums.storeLinks'], [ObjectType.Tag]: ['tags.id as tags.id', 'tags.name as tags.name', 'tags.parentId as tags.parentId'] }; function constructQuery(knex: Knex, userId: number, queryFor: ObjectType, queryElem: api.QueryElem, ordering: api.Ordering, offset: number, limit: number | null) { const joinObjects = getRequiredDatabaseObjects(queryElem); joinObjects.delete(queryFor); // We are already querying this object in the base query. // Figure out what data we want to select from the results. var columns: any[] = objectColumns[queryFor]; // TODO: there was a line here to add columns for the joined objects. // Could not get it to work with Postgres, which wants aggregate functions // to specify exactly how duplicates should be aggregated. // Not sure whether we need these columns in the first place. // joinObjects.forEach((obj: ObjectType) => columns.push(...objectColumns[obj])); // First, we create a base query for the type of object we need to yield. var q = knex.select(columns) .where({ [objectTables[queryFor] + '.user']: userId }) .groupBy(objectTables[queryFor] + '.' + 'id') .from(objectTables[queryFor]); // Now, we need to add join statements for other objects we want to filter on. joinObjects.forEach((object: ObjectType) => { q = addJoin(q, queryFor, object); }) // Apply filtering. q = q.andWhereRaw(getWhere(queryElem)); // Apply ordering const orderKeys = { [api.OrderByType.Name]: objectTables[queryFor] + '.' + ((queryFor === ObjectType.Track) ? 'name' : 'name') }; q = q.orderBy(orderKeys[ordering.orderBy.type], (ordering.ascending ? 'asc' : 'desc')); // Apply limiting. if (limit !== null) { q = q.limit(limit) } // Apply offsetting. q = q.offset(offset); return q; } async function getLinkedObjects(knex: Knex, userId: number, base: ObjectType, linked: ObjectType, baseIds: number[]) { var result: Record = {}; const table = objectTables[base]; const otherTable = objectTables[linked]; const maybeLinkingTable = getLinkingTable(base, linked); const maybeLinkingColumn = getLinkingColumn(base, linked); const columns = objectColumns[linked]; console.log(table, otherTable, maybeLinkingTable, maybeLinkingColumn); if (maybeLinkingTable) { await Promise.all(baseIds.map((baseId: number) => { return knex.select(columns).groupBy(otherTable + '.id').from(otherTable) .join(maybeLinkingTable, { [maybeLinkingTable + '.' + linkingTableIdNames[linked]]: otherTable + '.id' }) .where({ [otherTable + '.user']: userId }) .where({ [maybeLinkingTable + '.' + linkingTableIdNames[base]]: baseId }) .then((others: any) => { result[baseId] = others; }) })) } else if (maybeLinkingColumn) { await Promise.all(baseIds.map((baseId: number) => { return knex.select(columns).groupBy(otherTable + '.id').from(otherTable) .join(table, { [maybeLinkingColumn]: otherTable + '.id' }) .where({ [otherTable + '.user']: userId }) .where({ [table + '.id']: baseId }) .then((others: any) => { result[baseId] = others; }) })) } else { throw new Error('canno link objects.') } console.log("Query results for", baseIds, ":", result); return result; } // Resolve a tag into the full nested structure of its ancestors. async function getFullTag(knex: Knex, userId: number, tag: any): Promise { const resolveTag = async (t: any) => { if (t['tags.parentId']) { const parent = (await knex.select(objectColumns[ObjectType.Tag]) .from('tags') .where({ 'user': userId }) .where({ [objectTables[ObjectType.Tag] + '.id']: t['tags.parentId'] }))[0]; t.parent = await resolveTag(parent); } return t; } return await resolveTag(tag); } export async function doQuery(userId: number, q: api.QueryRequest, knex: Knex): Promise { const trackLimit = q.offsetsLimits.trackLimit; const trackOffset = q.offsetsLimits.trackOffset; const tagLimit = q.offsetsLimits.tagLimit; const tagOffset = q.offsetsLimits.tagOffset; const artistLimit = q.offsetsLimits.artistLimit; const artistOffset = q.offsetsLimits.artistOffset; const albumLimit = q.offsetsLimits.albumLimit; const albumOffset = q.offsetsLimits.albumOffset; const artistsPromise: Promise = (artistLimit && artistLimit !== 0) ? constructQuery(knex, userId, ObjectType.Artist, q.query, q.ordering, artistOffset || 0, artistLimit >= 0 ? artistLimit : null, ) : (async () => [])(); const albumsPromise: Promise = (albumLimit && albumLimit !== 0) ? constructQuery(knex, userId, ObjectType.Album, q.query, q.ordering, albumOffset || 0, albumLimit >= 0 ? albumLimit : null, ) : (async () => [])(); const tracksPromise: Promise = (trackLimit && trackLimit !== 0) ? constructQuery(knex, userId, ObjectType.Track, q.query, q.ordering, trackOffset || 0, trackLimit >= 0 ? trackLimit : null, ) : (async () => [])(); const tagsPromise: Promise = (tagLimit && tagLimit !== 0) ? constructQuery(knex, userId, ObjectType.Tag, q.query, q.ordering, tagOffset || 0, tagLimit >= 0 ? tagLimit : null, ) : (async () => [])(); // For some objects, we want to return linked information as well. // For that we need to do further queries. const trackIdsPromise = (async () => { const tracks = await tracksPromise; const ids = tracks.map((track: any) => track['tracks.id']); return ids; })(); const tracksArtistsPromise: Promise> = (trackLimit && trackLimit !== 0) ? (async () => { return await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Artist, await trackIdsPromise); })() : (async () => { return {}; })(); const tracksTagsPromise: Promise> = (trackLimit && trackLimit !== 0) ? (async () => { const tagsPerTrack: Record = await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Tag, await trackIdsPromise); var result: Record = {}; for (var key in tagsPerTrack) { const tags = tagsPerTrack[key]; var fullTags: any[] = []; for (var idx in tags) { fullTags.push(await getFullTag(knex, userId, tags[idx])); } result[key] = fullTags; } return result; })() : (async () => { return {}; })(); const tracksAlbumsPromise: Promise> = (trackLimit && trackLimit !== 0) ? (async () => { return await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Album, await trackIdsPromise); })() : (async () => { return {}; })(); const [ tracks, artists, albums, tags, tracksArtists, tracksTags, tracksAlbums, ] = await Promise.all([ tracksPromise, artistsPromise, albumsPromise, tagsPromise, tracksArtistsPromise, tracksTagsPromise, tracksAlbumsPromise, ]); var response: api.QueryResponse = { tracks: [], artists: [], albums: [], tags: [], }; switch (q.responseType) { case api.QueryResponseType.Details: { response = { tracks: tracks.map((track: any) => { const id = track['tracks.id']; return toApiTrack(track, tracksArtists[id], tracksTags[id], tracksAlbums[id]); }), artists: artists.map((artist: any) => { return toApiArtist(artist); }), albums: albums.map((album: any) => { return toApiAlbum(album); }), tags: tags.map((tag: any) => { return toApiTag(tag); }), }; break; } case api.QueryResponseType.Ids: { response = { tracks: tracks.map((track: any) => track['tracks.id']), artists: artists.map((artist: any) => artist['artists.id']), albums: albums.map((album: any) => album['albums.id']), tags: tags.map((tag: any) => tag['tags.id']), }; break; } case api.QueryResponseType.Count: { response = { tracks: tracks.length, artists: artists.length, albums: albums.length, tags: tags.length, }; break; } default: { throw new Error("Unimplemented response type.") } } console.log("Query response:", response) return response; }