MuDBase/server/endpoints/QueryEndpointHandler.ts

import * as api from '../../client/src/api';
import { EndpointError, EndpointHandler, catchUnhandledErrors } from './types';
import Knex from 'knex';
import asJson from '../lib/asJson';

enum ObjectType {
    Song = 0,
    Artist,
    Tag,
    Album,
}

// To keep track of which database objects are needed to filter on
// certain properties.
const propertyObjects: Record<api.QueryElemProperty, ObjectType> = {
    [api.QueryElemProperty.albumName]: ObjectType.Album,
    [api.QueryElemProperty.artistId]: ObjectType.Artist,
    [api.QueryElemProperty.artistName]: ObjectType.Artist,
    [api.QueryElemProperty.songId]: ObjectType.Song,
    [api.QueryElemProperty.songTitle]: ObjectType.Song,
}

// To keep track of the tables in which objects are stored.
const objectTables: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albums',
    [ObjectType.Artist]: 'artists',
    [ObjectType.Song]: 'songs',
    [ObjectType.Tag]: 'tags',
}

// To keep track of linking tables between objects.
const linkingTables: any = [
    [[ObjectType.Song, ObjectType.Album], 'songs_albums'],
    [[ObjectType.Song, ObjectType.Artist], 'songs_artists'],
    [[ObjectType.Song, ObjectType.Tag], 'songs_tags'],
    [[ObjectType.Artist, ObjectType.Album], 'artists_albums'],
    [[ObjectType.Artist, ObjectType.Tag], 'artists_tags'],
    [[ObjectType.Album, ObjectType.Tag], 'albums_tags'],
]
function getLinkingTable(a: ObjectType, b: ObjectType): string {
    var res: string | undefined = undefined;
    linkingTables.forEach((row: any) => {
        if (row[0].includes(a) && row[0].includes(b)) {
            res = row[1];
        }
    })
    if (res) return res;

    throw "Could not find linking table for objects: " + JSON.stringify(a) + ", " + JSON.stringify(b);
}

// To keep track of ID fields used in linking tables.
const linkingTableIdNames: Record<ObjectType, string> = {
    [ObjectType.Album]: 'albumId',
    [ObjectType.Artist]: 'artistId',
    [ObjectType.Song]: 'songId',
    [ObjectType.Tag]: 'tagId',
}

function getRequiredDatabaseObjects(queryElem: api.QueryElem): Set<ObjectType> {
    if (queryElem.prop) {
        // Leaf node.
        return new Set([propertyObjects[queryElem.prop]]);
    } else if (queryElem.children) {
        // Branch node.
        var r = new Set<ObjectType>();
        queryElem.children.forEach((child: api.QueryElem) => {
            getRequiredDatabaseObjects(child).forEach(object => r.add(object));
        });
        return r;
    }
    return new Set([]);
}

function addJoin(knexQuery: any, base: ObjectType, other: ObjectType) {
    const linkTable = getLinkingTable(base, other);
    const baseTable = objectTables[base];
    const otherTable = objectTables[other];
    return knexQuery
        .join(linkTable, { [baseTable + '.id']: linkTable + '.' + linkingTableIdNames[base] })
        .join(otherTable, { [otherTable + '.id']: linkTable + '.' + linkingTableIdNames[other] });
}

enum WhereType {
    And = 0,
    Or,
};

function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    const simpleLeafOps: Record<any, string> = {
        [api.QueryFilterOp.Eq]: "=",
        [api.QueryFilterOp.Ne]: "!=",
        [api.QueryFilterOp.Like]: "like",
    }

    const propertyKeys = {
        [api.QueryElemProperty.songTitle]: 'songs.title',
        [api.QueryElemProperty.songId]: 'songs.id',
        [api.QueryElemProperty.artistName]: 'artists.name',
        [api.QueryElemProperty.artistId]: 'artists.id',
        [api.QueryElemProperty.albumName]: 'albums.name',
    }

    if (!queryElem.propOperator) throw "Cannot create where clause without an operator.";
    const operator = queryElem.propOperator || api.QueryFilterOp.Eq;
    const a = queryElem.prop && propertyKeys[queryElem.prop];
    const b = queryElem.propOperand || "";

    if (Object.keys(simpleLeafOps).includes(operator)) {
        if (type == WhereType.And) {
            return knexQuery.andWhere(a, simpleLeafOps[operator], b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhere(a, simpleLeafOps[operator], b);
        }
    } else if (operator == api.QueryFilterOp.In) {
        if (type == WhereType.And) {
            return knexQuery.whereIn(a, b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhereIn(a, b);
        }
    } else if (operator == api.QueryFilterOp.NotIn) {
        if (type == WhereType.And) {
            return knexQuery.whereNotIn(a, b);
        } else if (type == WhereType.Or) {
            return knexQuery.orWhereNotIn(a, b);
        }
    }

    throw "Query filter not implemented";
}

function addBranchWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.And) {
        var q = knexQuery;
        queryElem.children.forEach((child: api.QueryElem) => {
            q = addWhere(q, child, type);
        })
        return q;
    } else if (queryElem.children && queryElem.childrenOperator === api.QueryElemOp.Or) {
        var q = knexQuery;
        const c = queryElem.children;
        const f = function (this: any) {
            for (var i = 0; i < c.length; i++) {
                addWhere(this, c[i], WhereType.Or);
            }
        }
        if (type == WhereType.And) {
            return q.where(f);
        } else if (type == WhereType.Or) {
            return q.orWhere(f);
        }
    }
}

function addWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) {
    if (queryElem.prop) {
        // Leaf node.
        return addLeafWhere(knexQuery, queryElem, type);
    } else if (queryElem.children) {
        // Branch node.
        return addBranchWhere(knexQuery, queryElem, type);
    }

    return knexQuery;
}

const objectColumns = {
    [ObjectType.Song]: ['songs.id as songs.id', 'songs.title as songs.title', 'songs.storeLinks as songs.storeLinks'],
    [ObjectType.Artist]: ['artists.id as artists.id', 'artists.name as artists.name', 'artists.storeLinks as artists.storeLinks'],
    [ObjectType.Album]: ['albums.id as albums.id', 'albums.name as albums.name', 'albums.storeLinks as albums.storeLinks'],
    [ObjectType.Tag]: ['tags.id as tags.id', 'tags.name as tags.name', 'tags.parentId as tags.parentId']
};

function constructQuery(knex: Knex, queryFor: ObjectType, queryElem: api.QueryElem, ordering: api.Ordering,
    offset: number, limit: number) {
    const joinObjects = getRequiredDatabaseObjects(queryElem);
    joinObjects.delete(queryFor); // We are already querying this object in the base query.

    // Figure out what data we want to select from the results.
    var columns: any[] = objectColumns[queryFor];

    // TODO: there was a line here to add columns for the joined objects.
    // Could not get it to work with Postgres, which wants aggregate functions
    // to specify exactly how duplicates should be aggregated.
    // Not sure whether we need these columns in the first place.
    // joinObjects.forEach((obj: ObjectType) => columns.push(...objectColumns[obj]));

    // First, we create a base query for the type of object we need to yield.
    var q = knex.select(columns)
        .groupBy(objectTables[queryFor] + '.' + 'id')
        .from(objectTables[queryFor]);

    // Now, we need to add join statements for other objects we want to filter on.
    joinObjects.forEach((object: ObjectType) => {
        q = addJoin(q, queryFor, object);
    })

    // Apply filtering.
    q = addWhere(q, queryElem, WhereType.And);

    // Apply ordering
    const orderKeys = {
        [api.OrderByType.Name]: objectTables[queryFor] + '.' + ((queryFor === ObjectType.Song) ? 'title' : 'name')
    };
    q = q.orderBy(orderKeys[ordering.orderBy.type],
        (ordering.ascending ? 'asc' : 'desc'));

    // Apply limiting.
    q = q.limit(limit).offset(offset);

    return q;
}

async function getLinkedObjects(knex: Knex, base: ObjectType, linked: ObjectType, baseIds: number[]) {
    var result: Record<number, any[]> = {};
    const otherTable = objectTables[linked];
    const linkingTable = getLinkingTable(base, linked);
    const columns = objectColumns[linked];

    await Promise.all(baseIds.map((baseId: number) => {
        return knex.select(columns).groupBy(otherTable + '.id').from(otherTable)
            .join(linkingTable, { [linkingTable + '.' + linkingTableIdNames[linked]]: otherTable + '.id' })
            .where({ [linkingTable + '.' + linkingTableIdNames[base]]: baseId })
            .then((others: any) => { result[baseId] = others; })
    }))

    console.log("Query results for", baseIds, ":", result);
    return result;
}

export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any, knex: Knex) => {
    if (!api.checkQueryRequest(req.body)) {
        const e: EndpointError = {
            internalMessage: 'Invalid Query request: ' + JSON.stringify(req.body),
            httpStatus: 400
        };
        throw e;
    }
    const reqObject: api.QueryRequest = req.body;
    console.log("Query: ", reqObject);

    try {
        const songLimit = reqObject.offsetsLimits.songLimit;
        const songOffset = reqObject.offsetsLimits.songOffset;
        const tagLimit = reqObject.offsetsLimits.tagLimit;
        const tagOffset = reqObject.offsetsLimits.tagOffset;
        const artistLimit = reqObject.offsetsLimits.artistLimit;
        const artistOffset = reqObject.offsetsLimits.artistOffset;

        const artistsPromise: Promise<any> = (artistLimit && artistLimit > 0) ?
            constructQuery(knex,
                ObjectType.Artist,
                reqObject.query,
                reqObject.ordering,
                artistOffset || 0,
                artistLimit
            ) :
            (async () => [])();

        const songsPromise: Promise<any> = (songLimit && songLimit > 0) ?
            constructQuery(knex,
                ObjectType.Song,
                reqObject.query,
                reqObject.ordering,
                songOffset || 0,
                songLimit
            ) :
            (async () => [])();

        const tagsPromise: Promise<any> = (tagLimit && tagLimit > 0) ?
            constructQuery(knex,
                ObjectType.Tag,
                reqObject.query,
                reqObject.ordering,
                tagOffset || 0,
                tagLimit
            ) :
            (async () => [])();

        // For some objects, we want to return linked information as well.
        // For that we need to do further queries.
        const songIdsPromise = (async () => {
            const songs = await songsPromise;
            console.log("Found songs:", songs);
            const ids = songs.map((song: any) => song['songs.id']);
            return ids;
        })();
        const songsArtistsPromise: Promise<Record<number, any[]>> = (songLimit && songLimit > 0) ?
            (async () => {
                return await getLinkedObjects(knex, ObjectType.Song, ObjectType.Artist, await songIdsPromise);
            })() :
            (async () => { return {}; })();
        const songsTagsPromise: Promise<Record<number, any[]>> = (songLimit && songLimit > 0) ?
            (async () => {
                return await getLinkedObjects(knex, ObjectType.Song, ObjectType.Tag, await songIdsPromise);
            })() :
            (async () => { return {}; })();

        const [
            songs,
            artists,
            tags,
            songsArtists,
            songsTags
        ] =
            await Promise.all([
                songsPromise,
                artistsPromise,
                tagsPromise,
                songsArtistsPromise,
                songsTagsPromise,
            ]);

        const response: api.QueryResponse = {
            songs: songs.map((song: any) => {
                return <api.SongDetails>{
                    songId: song['songs.id'],
                    title: song['songs.title'],
                    storeLinks: asJson(song['songs.storeLinks']),
                    artists: songsArtists[song['songs.id']].map((artist: any) => {
                        return <api.ArtistDetails>{
                            artistId: artist['artists.id'],
                            name: artist['artists.name'],
                            storeLinks: asJson(artist['artists.storeLinks']),
                        };
                    }),
                    tags: songsTags[song['songs.id']].map((tag: any) => {
                        return <api.TagDetails>{
                            tagId: tag['tags.id'],
                            name: tag['tags.name'],
                        };
                    }),
                    albums: [], //FIXME
                }
            }),
            artists: artists.map((artist: any) => {
                return <api.ArtistDetails>{
                    artistId: artist['artists.id'],
                    name: artist['artists.name'],
                    storeLinks: asJson(artist['artists.storeLinks']),
                }
            }),
            tags: tags.map((tag: any) => {
                return <api.TagDetails>{
                    tagId: tag['tags.id'],
                    name: tag['tags.name'],
                    parentId: tag['tags.parentId'],
                }
            }),
        }

        res.send(response);
    } catch (e) {
        catchUnhandledErrors(e);
    }

    // try {
    //     const songLimit = reqObject.offsetsLimits.songLimit;
    //     const songOffset = reqObject.offsetsLimits.songOffset;
    //     const tagLimit = reqObject.offsetsLimits.tagLimit;
    //     const tagOffset = reqObject.offsetsLimits.tagOffset;
    //     const artistLimit = reqObject.offsetsLimits.artistLimit;
    //     const artistOffset = reqObject.offsetsLimits.artistOffset;

    //     const songs = (songLimit && songLimit > 0) && await models.Song.findAll({
    //         // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
    //         // Custom pagination is implemented before responding.
    //         where: getSequelizeWhere(reqObject.query, QueryType.Song),
    //         order: getSequelizeOrder(reqObject.ordering, QueryType.Song),
    //         include: [ models.Artist, models.Album, models.Tag, models.Ranking ],
    //         //limit: reqObject.limit,
    //         //offset: reqObject.offset,
    //     })
    //     const artists = (artistLimit && artistLimit > 0) && await models.Artist.findAll({
    //         // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
    //         // Custom pagination is implemented before responding.
    //         where: getSequelizeWhere(reqObject.query, QueryType.Artist),
    //         order: getSequelizeOrder(reqObject.ordering, QueryType.Artist),
    //         include: [models.Song, models.Album, models.Tag],
    //         //limit: reqObject.limit,
    //         //offset: reqObject.offset,
    //     })
    //     const tags = (tagLimit && tagLimit > 0) && await models.Tag.findAll({
    //         // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
    //         // Custom pagination is implemented before responding.
    //         where: getSequelizeWhere(reqObject.query, QueryType.Tag),
    //         order: getSequelizeOrder(reqObject.ordering, QueryType.Tag),
    //         include: [models.Song, models.Album, models.Artist],
    //         //limit: reqObject.limit,
    //         //offset: reqObject.offset,
    //     })

    //     const response: api.QueryResponse = {
    //         songs: ((songLimit || -1) <= 0) ? [] : await Promise.all(songs.map(async (song: any) => {
    //             const artists = song.getArtists();
    //             const tags = song.getTags();
    //             const rankings = song.getRankings();
    //             return <api.SongDetails>{
    //                 songId: song.id,
    //                 title: song.title,
    //                 storeLinks: song.storeLinks,
    //                 artists: (await artists).map((artist: any) => {
    //                     return <api.ArtistDetails>{
    //                         artistId: artist.id,
    //                         name: artist.name,
    //                     }
    //                 }),
    //                 tags: (await tags).map((tag: any) => {
    //                     return <api.TagDetails>{
    //                         tagId: tag.id,
    //                         name: tag.name,
    //                     }
    //                 }),
    //                 rankings: await (await rankings).map(async (ranking: any) => {
    //                     const maybeTagContext: api.TagDetails | undefined = await ranking.getTagContext();
    //                     const maybeArtistContext: api.ArtistDetails | undefined = await ranking.getArtistContext();
    //                     const maybeContext = maybeTagContext || maybeArtistContext;
    //                     return <api.RankingDetails>{
    //                         rankingId: ranking.id,
    //                         type: api.ItemType.Song,
    //                         rankedId: song.id,
    //                         context: maybeContext,
    //                         value: ranking.value,
    //                     }
    //                 })
    //             };
    //         }).slice(songOffset || 0, (songOffset || 0) + (songLimit || 10))),
    //         // TODO: custom pagination due to bug mentioned above
    //         artists: ((artistLimit || -1) <= 0) ? [] : await Promise.all(artists.map(async (artist: any) => {
    //             return <api.ArtistDetails>{
    //                 artistId: artist.id,
    //                 name: artist.name,
    //             };
    //         }).slice(artistOffset || 0, (artistOffset || 0) + (artistLimit || 10))),
    //         tags: ((tagLimit || -1) <= 0) ? [] : await Promise.all(tags.map(async (tag: any) => {
    //             return <api.TagDetails>{
    //                 tagId: tag.id,
    //                 name: tag.name,
    //             };
    //         }).slice(tagOffset || 0,  (tagOffset || 0) + (tagLimit || 10))),
    //     };
    //     res.send(response);
    // } catch (e) {
    //     catchUnhandledErrors(e);
    // }
}