MuDBase/server/db/Query.ts

import * as api from '../../client/src/api/api';
import { EndpointError, EndpointHandler, handleErrorsInEndpoint } from '../endpoints/types';
import Knex from 'knex';
import asJson from '../lib/asJson';

export function toApiTag(dbObj: any): api.TagWithDetailsWithId {
    return <api.TagWithDetailsWithId>{
        mbApi_typename: "tag",
        id: dbObj['tags.id'],
        name: dbObj['tags.name'],
        parentId: dbObj['tags.parentId'],
        parent: dbObj.parent ? toApiTag(dbObj.parent) : undefined,
    };
}

export function toApiArtist(dbObj: any): api.ArtistWithId {
    return <api.ArtistWithId>{
        mbApi_typename: "artist",
        id: dbObj['artists.id'],
        name: dbObj['artists.name'],
        storeLinks: asJson(dbObj['artists.storeLinks']),
    };
}

export function toApiTrack(dbObj: any, artists: any[], tags: any[], albums: any[]): api.TrackWithDetailsWithId {
    return <api.TrackWithDetailsWithId>{
        mbApi_typename: "track",
        id: dbObj['tracks.id'],
        name: dbObj['tracks.name'],
        storeLinks: asJson(dbObj['tracks.storeLinks']),
        artists: artists.map((artist: any) => {
            return toApiArtist(artist);
        }),
        tags: tags.map((tag: any) => {
            return toApiTag(tag);
        }),
        album: albums.length > 0 ? toApiAlbum(albums[0]) : null,
    }
}

export function toApiAlbum(dbObj: any): api.AlbumWithId {
    return <api.AlbumWithId>{
        mbApi_typename: "album",
        id: dbObj['albums.id'],
        name: dbObj['albums.name'],
        storeLinks: asJson(dbObj['albums.storeLinks']),
    };
}

enum ObjectType {
    Track = 0,
    Artist,
    Tag,
    Album,
}

// To keep track of which database objects are needed to filter on
// certain properties.
const propertyObjects: Record<api.QueryElemProperty, ObjectType> = {
    [api.QueryElemProperty.albumId]: ObjectType.Album,
    [api.QueryElemProperty.albumName]: ObjectType.Album,
    [api.QueryElemProperty.artistId]: ObjectType.Artist,
    [api.QueryElemProperty.artistName]: ObjectType.Artist,
    [api.QueryElemProperty.trackId]: ObjectType.Track,
    [api.QueryElemProperty.trackName]: ObjectType.Track,
    [api.QueryElemProperty.tagId]: ObjectType.Tag,
    [api.QueryElemProperty.tagName]: ObjectType.Tag,
    [api.QueryElemProperty.trackStoreLinks]: ObjectType.Track,
    [api.QueryElemProperty.artistStoreLinks]: ObjectType.Artist,
    [api.QueryElemProperty.albumStoreLinks]: ObjectType.Album,
}

// To keep track of the tables in which objects are stored.
const objectTables: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albums',
    [ObjectType.Artist]: 'artists',
    [ObjectType.Track]: 'tracks',
    [ObjectType.Tag]: 'tags',
}

// To keep track of linking tables between objects.
const linkingTables: any = [
    [[ObjectType.Track, ObjectType.Artist], 'tracks_artists'],
    [[ObjectType.Track, ObjectType.Tag], 'tracks_tags'],
    [[ObjectType.Artist, ObjectType.Album], 'artists_albums'],
    [[ObjectType.Artist, ObjectType.Tag], 'artists_tags'],
    [[ObjectType.Album, ObjectType.Tag], 'albums_tags'],
]
function getLinkingTable(a: ObjectType, b: ObjectType): string | undefined {
    var res: string | undefined = undefined;
    linkingTables.forEach((row: any) => {
        if (row[0].includes(a) && row[0].includes(b)) {
            res = row[1];
        }
    })
    return res;
}

// To keep track of linking columns between objects.
const linkingColumns: any = [
    [[ObjectType.Track, ObjectType.Album], 'tracks.album'],
]
function getLinkingColumn(a: ObjectType, b: ObjectType): string | undefined {
    var res: string | undefined = undefined;
    linkingColumns.forEach((row: any) => {
        if (row[0].includes(a) && row[0].includes(b)) {
            res = row[1];
        }
    })
    return res;
}

// To keep track of ID fields used in linking tables.
const linkingTableIdNames: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albumId',
    [ObjectType.Artist]: 'artistId',
    [ObjectType.Track]: 'trackId',
    [ObjectType.Tag]: 'tagId',
}

function getRequiredDatabaseObjects(queryElem: api.QueryElem): Set<ObjectType> {
    if (queryElem.prop) {
        // Leaf node.
        return new Set([propertyObjects[queryElem.prop]]);
    } else if (queryElem.children) {
        // Branch node.
        var r = new Set<ObjectType>();
        queryElem.children.forEach((child: api.QueryElem) => {
            getRequiredDatabaseObjects(child).forEach(object => r.add(object));
        });
        return r;
    }
    return new Set([]);
}

function addJoin(knexQuery: any, base: ObjectType, other: ObjectType) {
    const linkTable = getLinkingTable(base, other);
    const linkColumn = getLinkingColumn(base, other);
    const baseTable = objectTables[base];
    const otherTable = objectTables[other];

    if (linkTable) {
        return knexQuery
            .join(linkTable, { [baseTable + '.id']: linkTable + '.' + linkingTableIdNames[base] })
            .join(otherTable, { [otherTable + '.id']: linkTable + '.' + linkingTableIdNames[other] });
    } else if (linkColumn) {
        return knexQuery
            .join(otherTable, { [linkColumn]: otherTable + '.id' });
    }
}

enum WhereType {
    And = 0,
    Or,
};

function getSQLValue(val: any) {
    console.log("Value:", val)
    if (typeof val === 'string') {
        return `'${val}'`;
    } else if (typeof val === 'number') {
        return `${val}`;
    }
    throw new Error("unimplemented SQL value type.");
}

function getSQLValues(vals: any[]) {
    if (vals.length === 0) { return '()' }
    let r = `(${getSQLValue(vals[0])}`;
    for (let i: number = 1; i < vals.length; i++) {
        r += `, ${getSQLValue(vals[i])}`;
    }
    r += ')';
    return r;
}

function getLeafWhere(queryElem: api.QueryElem): string {
    const simpleLeafOps: Record<any, string> = {
        [api.QueryLeafOp.Eq]: "=",
        [api.QueryLeafOp.Ne]: "!=",
        [api.QueryLeafOp.Like]: "LIKE",
    }

    const propertyKeys = {
        [api.QueryElemProperty.trackName]: '`tracks`.`name`',
        [api.QueryElemProperty.trackId]: '`tracks`.`id`',
        [api.QueryElemProperty.artistName]: '`artists`.`name`',
        [api.QueryElemProperty.artistId]: '`artists`.`id`',
        [api.QueryElemProperty.albumName]: '`albums`.`name`',
        [api.QueryElemProperty.albumId]: '`albums`.`id`',
        [api.QueryElemProperty.tagId]: '`tags`.`id`',
        [api.QueryElemProperty.tagName]: '`tags`.`name`',
        [api.QueryElemProperty.trackStoreLinks]: '`tracks`.`storeLinks`',
        [api.QueryElemProperty.artistStoreLinks]: '`artists`.`storeLinks`',
        [api.QueryElemProperty.albumStoreLinks]: '`albums`.`storeLinks`',
    }

    if (!queryElem.propOperator) throw "Cannot create where clause without an operator.";
    const operator = queryElem.propOperator || api.QueryLeafOp.Eq;
    const a = queryElem.prop && propertyKeys[queryElem.prop];
    const b = operator === api.QueryLeafOp.Like ?
        '%' + (queryElem.propOperand || "") + '%'
        : (queryElem.propOperand || "");

    if (Object.keys(simpleLeafOps).includes(operator)) {
        return `(${a} ${simpleLeafOps[operator]} ${getSQLValue(b)})`;
    } else if (operator == api.QueryLeafOp.In) {
        return `(${a} IN ${getSQLValues(b)})`
    } else if (operator == api.QueryLeafOp.NotIn) {
        return `(${a} NOT IN ${getSQLValues(b)})`
    }

    throw "Query filter not implemented";
}

function getNodeWhere(queryElem: api.QueryElem): string {
    let ops = {
        [api.QueryNodeOp.And]: 'AND',
        [api.QueryNodeOp.Or]: 'OR',
        [api.QueryNodeOp.Not]: 'NOT',
    }
    let buildList = (subqueries: api.QueryElem[], operator: api.QueryNodeOp) => {
        if (subqueries.length === 0) { return 'true' }
        let r = `(${getWhere(subqueries[0])}`;
        for (let i: number = 1; i < subqueries.length; i++) {
            r += ` ${ops[operator]} ${getWhere(subqueries[i])}`;
        }
        r += ')';
        return r;
    }

    if (queryElem.children && queryElem.childrenOperator && queryElem.children.length) {
        if (queryElem.childrenOperator === api.QueryNodeOp.And ||
            queryElem.childrenOperator === api.QueryNodeOp.Or) {
            return buildList(queryElem.children, queryElem.childrenOperator)
        } else if (queryElem.childrenOperator === api.QueryNodeOp.Not &&
            queryElem.children.length === 1) {
            return `NOT ${getWhere(queryElem.children[0])}`
        }
    }

    throw new Error('invalid query')
}

function getWhere(queryElem: api.QueryElem): string {
    if (queryElem.prop) { return getLeafWhere(queryElem); }
    if (queryElem.children) { return getNodeWhere(queryElem); }
    return "true";
}

const objectColumns = {
    [ObjectType.Track]: ['tracks.id as tracks.id', 'tracks.name as tracks.name', 'tracks.storeLinks as tracks.storeLinks', 'tracks.album as tracks.album'],
    [ObjectType.Artist]: ['artists.id as artists.id', 'artists.name as artists.name', 'artists.storeLinks as artists.storeLinks'],
    [ObjectType.Album]: ['albums.id as albums.id', 'albums.name as albums.name', 'albums.storeLinks as albums.storeLinks'],
    [ObjectType.Tag]: ['tags.id as tags.id', 'tags.name as tags.name', 'tags.parentId as tags.parentId']
};

function constructQuery(knex: Knex, userId: number, queryFor: ObjectType, queryElem: api.QueryElem, ordering: api.Ordering,
    offset: number, limit: number | null) {
    const joinObjects = getRequiredDatabaseObjects(queryElem);
    joinObjects.delete(queryFor); // We are already querying this object in the base query.

    // Figure out what data we want to select from the results.
    var columns: any[] = objectColumns[queryFor];

    // TODO: there was a line here to add columns for the joined objects.
    // Could not get it to work with Postgres, which wants aggregate functions
    // to specify exactly how duplicates should be aggregated.
    // Not sure whether we need these columns in the first place.
    // joinObjects.forEach((obj: ObjectType) => columns.push(...objectColumns[obj]));

    // First, we create a base query for the type of object we need to yield.
    var q = knex.select(columns)
        .where({ [objectTables[queryFor] + '.user']: userId })
        .groupBy(objectTables[queryFor] + '.' + 'id')
        .from(objectTables[queryFor]);

    // Now, we need to add join statements for other objects we want to filter on.
    joinObjects.forEach((object: ObjectType) => {
        q = addJoin(q, queryFor, object);
    })

    // Apply filtering.
    q = q.andWhereRaw(getWhere(queryElem));

    // Apply ordering
    const orderKeys = {
        [api.OrderByType.Name]: objectTables[queryFor] + '.' + ((queryFor === ObjectType.Track) ? 'name' : 'name')
    };
    q = q.orderBy(orderKeys[ordering.orderBy.type],
        (ordering.ascending ? 'asc' : 'desc'));

    // Apply limiting.
    if (limit !== null) {
        q = q.limit(limit)
    }

    // Apply offsetting.
    q = q.offset(offset);

    return q;
}

async function getLinkedObjects(knex: Knex, userId: number, base: ObjectType, linked: ObjectType, baseIds: number[]) {
    var result: Record<number, any[]> = {};
    const table = objectTables[base];
    const otherTable = objectTables[linked];
    const maybeLinkingTable = getLinkingTable(base, linked);
    const maybeLinkingColumn = getLinkingColumn(base, linked);
    const columns = objectColumns[linked];

    console.log(table, otherTable, maybeLinkingTable, maybeLinkingColumn);

    if (maybeLinkingTable) {
        await Promise.all(baseIds.map((baseId: number) => {
            return knex.select(columns).groupBy(otherTable + '.id').from(otherTable)
                .join(maybeLinkingTable, { [maybeLinkingTable + '.' + linkingTableIdNames[linked]]: otherTable + '.id' })
                .where({ [otherTable + '.user']: userId })
                .where({ [maybeLinkingTable + '.' + linkingTableIdNames[base]]: baseId })
                .then((others: any) => { result[baseId] = others; })
        }))
    } else if (maybeLinkingColumn) {
        await Promise.all(baseIds.map((baseId: number) => {
            return knex.select(columns).groupBy(otherTable + '.id').from(otherTable)
                .join(table, { [maybeLinkingColumn]: otherTable + '.id' })
                .where({ [otherTable + '.user']: userId })
                .where({ [table + '.id']: baseId })
                .then((others: any) => { result[baseId] = others; })
        }))
    } else {
        throw new Error('canno link objects.')
    }

    console.log("Query results for", baseIds, ":", result);
    return result;
}

// Resolve a tag into the full nested structure of its ancestors.
async function getFullTag(knex: Knex, userId: number, tag: any): Promise<any> {
    const resolveTag = async (t: any) => {
        if (t['tags.parentId']) {
            const parent = (await knex.select(objectColumns[ObjectType.Tag])
                .from('tags')
                .where({ 'user': userId })
                .where({ [objectTables[ObjectType.Tag] + '.id']: t['tags.parentId'] }))[0];
            t.parent = await resolveTag(parent);
        }
        return t;
    }

    return await resolveTag(tag);
}

export async function doQuery(userId: number, q: api.QueryRequest, knex: Knex): Promise<api.QueryResponse> {
    const trackLimit = q.offsetsLimits.trackLimit;
    const trackOffset = q.offsetsLimits.trackOffset;
    const tagLimit = q.offsetsLimits.tagLimit;
    const tagOffset = q.offsetsLimits.tagOffset;
    const artistLimit = q.offsetsLimits.artistLimit;
    const artistOffset = q.offsetsLimits.artistOffset;
    const albumLimit = q.offsetsLimits.albumLimit;
    const albumOffset = q.offsetsLimits.albumOffset;

    const artistsPromise: Promise<any> = (artistLimit && artistLimit !== 0) ?
        constructQuery(knex,
            userId,
            ObjectType.Artist,
            q.query,
            q.ordering,
            artistOffset || 0,
            artistLimit >= 0 ? artistLimit : null,
        ) :
        (async () => [])();

    const albumsPromise: Promise<any> = (albumLimit && albumLimit !== 0) ?
        constructQuery(knex,
            userId,
            ObjectType.Album,
            q.query,
            q.ordering,
            albumOffset || 0,
            albumLimit >= 0 ? albumLimit : null,
        ) :
        (async () => [])();

    const tracksPromise: Promise<any> = (trackLimit && trackLimit !== 0) ?
        constructQuery(knex,
            userId,
            ObjectType.Track,
            q.query,
            q.ordering,
            trackOffset || 0,
            trackLimit >= 0 ? trackLimit : null,
        ) :
        (async () => [])();

    const tagsPromise: Promise<any> = (tagLimit && tagLimit !== 0) ?
        constructQuery(knex,
            userId,
            ObjectType.Tag,
            q.query,
            q.ordering,
            tagOffset || 0,
            tagLimit >= 0 ? tagLimit : null,
        ) :
        (async () => [])();

    // For some objects, we want to return linked information as well.
    // For that we need to do further queries.
    const trackIdsPromise = (async () => {
        const tracks = await tracksPromise;
        const ids = tracks.map((track: any) => track['tracks.id']);
        return ids;
    })();
    const tracksArtistsPromise: Promise<Record<number, any[]>> = (trackLimit && trackLimit !== 0) ?
        (async () => {
            return await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Artist, await trackIdsPromise);
        })() :
        (async () => { return {}; })();
    const tracksTagsPromise: Promise<Record<number, any[]>> = (trackLimit && trackLimit !== 0) ?
        (async () => {
            const tagsPerTrack: Record<number, any> = await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Tag, await trackIdsPromise);
            var result: Record<number, any> = {};
            for (var key in tagsPerTrack) {
                const tags = tagsPerTrack[key];
                var fullTags: any[] = [];
                for (var idx in tags) {
                    fullTags.push(await getFullTag(knex, userId, tags[idx]));
                }
                result[key] = fullTags;
            }
            return result;
        })() :
        (async () => { return {}; })();
    const tracksAlbumsPromise: Promise<Record<number, any[]>> = (trackLimit && trackLimit !== 0) ?
        (async () => {
            return await getLinkedObjects(knex, userId, ObjectType.Track, ObjectType.Album, await trackIdsPromise);
        })() :
        (async () => { return {}; })();

    const [
        tracks,
        artists,
        albums,
        tags,
        tracksArtists,
        tracksTags,
        tracksAlbums,
    ] =
        await Promise.all([
            tracksPromise,
            artistsPromise,
            albumsPromise,
            tagsPromise,
            tracksArtistsPromise,
            tracksTagsPromise,
            tracksAlbumsPromise,
        ]);

    var response: api.QueryResponse = {
        tracks: [],
        artists: [],
        albums: [],
        tags: [],
    };

    switch (q.responseType) {
        case api.QueryResponseType.Details: {
            response = {
                tracks: tracks.map((track: any) => {
                    const id = track['tracks.id'];
                    return toApiTrack(track, tracksArtists[id], tracksTags[id], tracksAlbums[id]);
                }),
                artists: artists.map((artist: any) => {
                    return toApiArtist(artist);
                }),
                albums: albums.map((album: any) => {
                    return toApiAlbum(album);
                }),
                tags: tags.map((tag: any) => {
                    return toApiTag(tag);
                }),
            };
            break;
        }
        case api.QueryResponseType.Ids: {
            response = {
                tracks: tracks.map((track: any) => track['tracks.id']),
                artists: artists.map((artist: any) => artist['artists.id']),
                albums: albums.map((album: any) => album['albums.id']),
                tags: tags.map((tag: any) => tag['tags.id']),
            };
            break;
        }
        case api.QueryResponseType.Count: {
            response = {
                tracks: tracks.length,
                artists: artists.length,
                albums: albums.length,
                tags: tags.length,
            };
            break;
        }
        default: {
            throw new Error("Unimplemented response type.")
        }
    }

    console.log("Query response:", response)
    return response;
}