MuDBase/server/endpoints/Query.ts

import * as api from '../../client/src/api';
import { EndpointError, EndpointHandler, catchUnhandledErrors } from './types';
import Knex from 'knex';
import asJson from '../lib/asJson';
import { toApiArtist, toApiTag, toApiAlbum, toApiSong } from '../lib/dbToApi';

enum ObjectType {
    Song = 0,
    Artist,
    Tag,
    Album,
}

// To keep track of which database objects are needed to filter on
// certain properties.
const propertyObjects: Record<api.QueryElemProperty, ObjectType> = {
    [api.QueryElemProperty.albumId]: ObjectType.Album,
    [api.QueryElemProperty.albumName]: ObjectType.Album,
    [api.QueryElemProperty.artistId]: ObjectType.Artist,
    [api.QueryElemProperty.artistName]: ObjectType.Artist,
    [api.QueryElemProperty.songId]: ObjectType.Song,
    [api.QueryElemProperty.songTitle]: ObjectType.Song,
    [api.QueryElemProperty.tagId]: ObjectType.Tag,
}

// To keep track of the tables in which objects are stored.
const objectTables: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albums',
    [ObjectType.Artist]: 'artists',
    [ObjectType.Song]: 'songs',
    [ObjectType.Tag]: 'tags',
}

// To keep track of linking tables between objects.
const linkingTables: any = [
    [[ObjectType.Song, ObjectType.Album], 'songs_albums'],
    [[ObjectType.Song, ObjectType.Artist], 'songs_artists'],
    [[ObjectType.Song, ObjectType.Tag], 'songs_tags'],
    [[ObjectType.Artist, ObjectType.Album], 'artists_albums'],
    [[ObjectType.Artist, ObjectType.Tag], 'artists_tags'],
    [[ObjectType.Album, ObjectType.Tag], 'albums_tags'],
]
function getLinkingTable(a: ObjectType, b: ObjectType): string {
    var res: string | undefined = undefined;
    linkingTables.forEach((row: any) => {
        if (row[0].includes(a) && row[0].includes(b)) {
            res = row[1];
        }
    })
    if (res) return res;

    throw "Could not find linking table for objects: " + JSON.stringify(a) + ", " + JSON.stringify(b);
}

// To keep track of ID fields used in linking tables.
const linkingTableIdNames: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albumId',
    [ObjectType.Artist]: 'artistId',
    [ObjectType.Song]: 'songId',
    [ObjectType.Tag]: 'tagId',
}

function getRequiredDatabaseObjects(queryElem: api.QueryElem): Set<ObjectType> {
    if (queryElem.prop) {
        // Leaf node.
        return new Set([propertyObjects[queryElem.prop]]);
    } else if (queryElem.children) {
        // Branch node.
        var r = new Set<ObjectType>();
        queryElem.children.forEach((child: api.QueryElem) => {
            getRequiredDatabaseObjects(child).forEach(object => r.add(object));
        });
        return r;
    }
    return new Set([]);
}

function addJoin(knexQuery: any, base: ObjectType, other: ObjectType) {
    const linkTable = getLinkingTable(base, other);
    const baseTable = objectTables[base];
    const otherTable = objectTables[other];
    return knexQuery
        .join(linkTable, { [baseTable + '.id']: linkTable + '.' + linkingTableIdNames[base] })
        .join(otherTable, { [otherTable + '.id']: linkTable + '.' + linkingTableIdNames[other] });
}

enum WhereType {
    And = 0,
    Or,
};

function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    const simpleLeafOps: Record<any, string> = {
        [api.QueryFilterOp.Eq]: "=",
        [api.QueryFilterOp.Ne]: "!=",
        [api.QueryFilterOp.Like]: "like",
    }

    const propertyKeys = {
        [api.QueryElemProperty.songTitle]: 'songs.title',
        [api.QueryElemProperty.songId]: 'songs.id',
        [api.QueryElemProperty.artistName]: 'artists.name',
        [api.QueryElemProperty.artistId]: 'artists.id',
        [api.QueryElemProperty.albumName]: 'albums.name',
        [api.QueryElemProperty.albumId]: 'albums.id',
        [api.QueryElemProperty.tagId]: 'tags.id',
    }

    if (!queryElem.propOperator) throw "Cannot create where clause without an operator.";
    const operator = queryElem.propOperator || api.QueryFilterOp.Eq;
    const a = queryElem.prop && propertyKeys[queryElem.prop];
    const b = operator === api.QueryFilterOp.Like ?
        '%' + (queryElem.propOperand || "") + '%'
        : (queryElem.propOperand || "");

    if (Object.keys(simpleLeafOps).includes(operator)) {
        if (type == WhereType.And) {
            return knexQuery.andWhere(a, simpleLeafOps[operator], b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhere(a, simpleLeafOps[operator], b);
        }
    } else if (operator == api.QueryFilterOp.In) {
        if (type == WhereType.And) {
            return knexQuery.whereIn(a, b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhereIn(a, b);
        }
    } else if (operator == api.QueryFilterOp.NotIn) {
        if (type == WhereType.And) {
            return knexQuery.whereNotIn(a, b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhereNotIn(a, b);
        }
    }

    throw "Query filter not implemented";
}

function addBranchWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.And) {
        var q = knexQuery;
        queryElem.children.forEach((child: api.QueryElem) => {
            q = addWhere(q, child, type);
        })
        return q;
    } else if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.Or) {
        var q = knexQuery;
        const c = queryElem.children;
        const f = function (this: any) {
            for (var i = 0; i < c.length; i++) {
                addWhere(this, c[i], WhereType.Or);
            }
        }
        if (type == WhereType.And) {
            return q.where(f);
        } else if (type == WhereType.Or) {
            return q.orWhere(f);
        }
    }
}

function addWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    if (queryElem.prop) {
        // Leaf node.
        return addLeafWhere(knexQuery, queryElem, type);
    } else if (queryElem.children) {
        // Branch node.
        return addBranchWhere(knexQuery, queryElem, type);
    }

    return knexQuery;
}

const objectColumns = {
    [ObjectType.Song]: ['songs.id as songs.id', 'songs.title as songs.title', 'songs.storeLinks as songs.storeLinks'],
    [ObjectType.Artist]: ['artists.id as artists.id', 'artists.name as artists.name', 'artists.storeLinks as artists.storeLinks'],
    [ObjectType.Album]: ['albums.id as albums.id', 'albums.name as albums.name', 'albums.storeLinks as albums.storeLinks'],
    [ObjectType.Tag]: ['tags.id as tags.id', 'tags.name as tags.name', 'tags.parentId as tags.parentId']
};

function constructQuery(knex: Knex, userId: number, queryFor: ObjectType, queryElem: api.QueryElem, ordering: api.Ordering,
    offset: number, limit: number | null) {
    const joinObjects = getRequiredDatabaseObjects(queryElem);
    joinObjects.delete(queryFor); // We are already querying this object in the base query.

    // Figure out what data we want to select from the results.
    var columns: any[] = objectColumns[queryFor];

    // TODO: there was a line here to add columns for the joined objects.
    // Could not get it to work with Postgres, which wants aggregate functions
    // to specify exactly how duplicates should be aggregated.
    // Not sure whether we need these columns in the first place.
    // joinObjects.forEach((obj: ObjectType) => columns.push(...objectColumns[obj]));

    // First, we create a base query for the type of object we need to yield.
    var q = knex.select(columns)
        .where({ [objectTables[queryFor] + '.user']: userId })
        .groupBy(objectTables[queryFor] + '.' + 'id')
        .from(objectTables[queryFor]);

    // Now, we need to add join statements for other objects we want to filter on.
    joinObjects.forEach((object: ObjectType) => {
        q = addJoin(q, queryFor, object);
    })

    // Apply filtering.
    q = addWhere(q, queryElem, WhereType.And);

    // Apply ordering
    const orderKeys = {
        [api.OrderByType.Name]: objectTables[queryFor] + '.' + ((queryFor === ObjectType.Song) ? 'title' : 'name')
    };
    q = q.orderBy(orderKeys[ordering.orderBy.type],
        (ordering.ascending ? 'asc' : 'desc'));

    // Apply limiting.
    if(limit !== null) {
        q = q.limit(limit)
    }

    // Apply offsetting.
    q = q.offset(offset);

    return q;
}

async function getLinkedObjects(knex: Knex, userId: number, base: ObjectType, linked: ObjectType, baseIds: number[]) {
    var result: Record<number, any[]> = {};
    const otherTable = objectTables[linked];
    const linkingTable = getLinkingTable(base, linked);
    const columns = objectColumns[linked];

    await Promise.all(baseIds.map((baseId: number) => {
        return knex.select(columns).groupBy(otherTable + '.id').from(otherTable)
            .join(linkingTable, { [linkingTable + '.' + linkingTableIdNames[linked]]: otherTable + '.id' })
            .where({ [otherTable + '.user']: userId })
            .where({ [linkingTable + '.' + linkingTableIdNames[base]]: baseId })
            .then((others: any) => { result[baseId] = others; })
    }))

    console.log("Query results for", baseIds, ":", result);
    return result;
}

// Resolve a tag into the full nested structure of its ancestors.
async function getFullTag(knex: Knex, userId: number, tag: any): Promise<any> {
    const resolveTag = async (t: any) => {
        if (t['tags.parentId']) {
            const parent = (await knex.select(objectColumns[ObjectType.Tag])
                .from('tags')
                .where({ 'user': userId })
                .where({ [objectTables[ObjectType.Tag] + '.id']: t['tags.parentId'] }))[0];
            t.parent = await resolveTag(parent);
        }
        return t;
    }

    return await resolveTag(tag);
}

export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any, knex: Knex) => {
    if (!api.checkQueryRequest(req.body)) {
        const e: EndpointError = {
            internalMessage: 'Invalid Query request: ' + JSON.stringify(req.body),
            httpStatus: 400
        };
        throw e;
    }
    const reqObject: api.QueryRequest = req.body;
    const { id: userId } = req.user;

    console.log("User ", userId, ": Query ", reqObject);

    try {
        const songLimit = reqObject.offsetsLimits.songLimit;
        const songOffset = reqObject.offsetsLimits.songOffset;
        const tagLimit = reqObject.offsetsLimits.tagLimit;
        const tagOffset = reqObject.offsetsLimits.tagOffset;
        const artistLimit = reqObject.offsetsLimits.artistLimit;
        const artistOffset = reqObject.offsetsLimits.artistOffset;
        const albumLimit = reqObject.offsetsLimits.albumLimit;
        const albumOffset = reqObject.offsetsLimits.albumOffset;

        const artistsPromise: Promise<any> = (artistLimit && artistLimit !== 0) ?
            constructQuery(knex,
                userId,
                ObjectType.Artist,
                reqObject.query,
                reqObject.ordering,
                artistOffset || 0,
                artistLimit >= 0 ? artistLimit : null,
            ) :
            (async () => [])();

        const albumsPromise: Promise<any> = (albumLimit && albumLimit !== 0) ?
            constructQuery(knex,
                userId,
                ObjectType.Album,
                reqObject.query,
                reqObject.ordering,
                artistOffset || 0,
                albumLimit >= 0 ? albumLimit : null,
            ) :
            (async () => [])();

        const songsPromise: Promise<any> = (songLimit && songLimit !== 0) ?
            constructQuery(knex,
                userId,
                ObjectType.Song,
                reqObject.query,
                reqObject.ordering,
                songOffset || 0,
                songLimit >= 0 ? songLimit : null,
            ) :
            (async () => [])();

        const tagsPromise: Promise<any> = (tagLimit && tagLimit !== 0) ?
            constructQuery(knex,
                userId,
                ObjectType.Tag,
                reqObject.query,
                reqObject.ordering,
                tagOffset || 0,
                tagLimit >= 0 ? tagLimit : null,
            ) :
            (async () => [])();

        // For some objects, we want to return linked information as well.
        // For that we need to do further queries.
        const songIdsPromise = (async () => {
            const songs = await songsPromise;
            const ids = songs.map((song: any) => song['songs.id']);
            return ids;
        })();
        const songsArtistsPromise: Promise<Record<number, any[]>> = (songLimit && songLimit !== 0) ?
            (async () => {
                return await getLinkedObjects(knex, userId, ObjectType.Song, ObjectType.Artist, await songIdsPromise);
            })() :
            (async () => { return {}; })();
        const songsTagsPromise: Promise<Record<number, any[]>> = (songLimit && songLimit !== 0) ?
            (async () => {
                const tagsPerSong: Record<number, any> = await getLinkedObjects(knex, userId, ObjectType.Song, ObjectType.Tag, await songIdsPromise);
                var result: Record<number, any> = {};
                for (var key in tagsPerSong) {
                    const tags = tagsPerSong[key];
                    var fullTags: any[] = [];
                    for (var idx in tags) {
                        fullTags.push(await getFullTag(knex, userId, tags[idx]));
                    }
                    result[key] = fullTags;
                }
                return result;
            })() :
            (async () => { return {}; })();
        const songsAlbumsPromise: Promise<Record<number, any[]>> = (songLimit && songLimit !== 0) ?
            (async () => {
                return await getLinkedObjects(knex, userId, ObjectType.Song, ObjectType.Album, await songIdsPromise);
            })() :
            (async () => { return {}; })();

        const [
            songs,
            artists,
            albums,
            tags,
            songsArtists,
            songsTags,
            songsAlbums,
        ] =
            await Promise.all([
                songsPromise,
                artistsPromise,
                albumsPromise,
                tagsPromise,
                songsArtistsPromise,
                songsTagsPromise,
                songsAlbumsPromise,
            ]);

        const response: api.QueryResponse = {
            songs: songs.map((song: any) => {
                const id = song['songs.id'];
                return toApiSong(song, songsArtists[id], songsTags[id], songsAlbums[id]);
            }),
            artists: artists.map((artist: any) => {
                return toApiArtist(artist);
            }),
            albums: albums.map((album: any) => {
                return toApiAlbum(album);
            }),
            tags: tags.map((tag: any) => {
                return toApiTag(tag);
            }),
        }

        console.log("Query repsonse", response);

        res.send(response);
    } catch (e) {
        catchUnhandledErrors(e);
    }
}