From 1148bc6bfff035c725dbe6e1283a6f50fb3f5f1b Mon Sep 17 00:00:00 2001 From: Sander Vocke Date: Thu, 26 Nov 2020 09:36:05 +0100 Subject: [PATCH] Change storeLinks storage to text. Make it queriable. --- client/src/api.ts | 3 + scripts/gpm_retrieve/gpm_retrieve.py | 2 +- server/endpoints/Query.ts | 6 ++ server/knex/get_knex.ts | 3 +- .../20201126082705_storelinks_to_text.ts | 58 +++++++++++++++++++ server/test/integration/flows/QueryFlow.js | 36 +++++++++++- 6 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 server/migrations/20201126082705_storelinks_to_text.ts diff --git a/client/src/api.ts b/client/src/api.ts index c872f29..99733cb 100644 --- a/client/src/api.ts +++ b/client/src/api.ts @@ -83,6 +83,9 @@ export enum QueryElemProperty { albumName = "albumName", albumId = "albumId", tagId = "tagId", + songStoreLinks = "songStoreLinks", //Note: treated as a JSON string for filter operations + artistStoreLinks = "artistStoreLinks", //Note: treated as a JSON string for filter operations + albumStoreLinks = "albumStoreLinks", //Note: treated as a JSON string for filter operations } export enum OrderByType { Name = 0, diff --git a/scripts/gpm_retrieve/gpm_retrieve.py b/scripts/gpm_retrieve/gpm_retrieve.py index 87fc9f3..7efa4e9 100755 --- a/scripts/gpm_retrieve/gpm_retrieve.py +++ b/scripts/gpm_retrieve/gpm_retrieve.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import Mobileclient from gmusicapi +from gmusicapi import Mobileclient import argparse import sys import requests diff --git a/server/endpoints/Query.ts b/server/endpoints/Query.ts index f705516..36715ec 100644 --- a/server/endpoints/Query.ts +++ b/server/endpoints/Query.ts @@ -21,6 +21,9 @@ const propertyObjects: Record = { [api.QueryElemProperty.songId]: ObjectType.Song, [api.QueryElemProperty.songTitle]: ObjectType.Song, [api.QueryElemProperty.tagId]: ObjectType.Tag, + [api.QueryElemProperty.songStoreLinks]: ObjectType.Song, + [api.QueryElemProperty.artistStoreLinks]: ObjectType.Artist, + [api.QueryElemProperty.albumStoreLinks]: ObjectType.Album, } // To keep track of the tables in which objects are stored. @@ -104,6 +107,9 @@ function addLeafWhere(knexQuery: any, queryElem: api.QueryElem, type: WhereType) [api.QueryElemProperty.albumName]: 'albums.name', [api.QueryElemProperty.albumId]: 'albums.id', [api.QueryElemProperty.tagId]: 'tags.id', + [api.QueryElemProperty.songStoreLinks]: 'songs.storeLinks', + [api.QueryElemProperty.artistStoreLinks]: 'artists.storeLinks', + [api.QueryElemProperty.albumStoreLinks]: 'albums.storeLinks', } if (!queryElem.propOperator) throw "Cannot create where clause without an operator."; diff --git a/server/knex/get_knex.ts b/server/knex/get_knex.ts index 05d06f2..bb804ac 100644 --- a/server/knex/get_knex.ts +++ b/server/knex/get_knex.ts @@ -1,7 +1,8 @@ const environment = process.env.ENVIRONMENT || 'development' +import Knex from 'knex'; import config from '../knexfile'; -export default function get_knex() { +export default function get_knex(): Knex { if (!Object.keys(config).includes(environment)) { throw "No Knex database configuration was found for environment '" + environment + "'. Please check your configuration."; diff --git a/server/migrations/20201126082705_storelinks_to_text.ts b/server/migrations/20201126082705_storelinks_to_text.ts new file mode 100644 index 0000000..ac174ee --- /dev/null +++ b/server/migrations/20201126082705_storelinks_to_text.ts @@ -0,0 +1,58 @@ +import * as Knex from "knex"; + +/* +This migration converts the storeLinks column from JSON to plain text. +The reason is that there are too many differences between the JSON support +of different back-ends, making plain text easier to deal with. +*/ + +async function castStoreLinks(table: string, knex: any) { + await knex.schema.alterTable(table, (t: any) => { + t.string('storeLinksTemp'); + }); + await knex(table).update({ + storeLinksTemp: knex.raw('CAST("storeLinks" AS VARCHAR(255))') + }) + await knex.schema.alterTable(table, (t: any) => { + t.dropColumn('storeLinks'); + }); + await knex.schema.alterTable(table, (t: any) => { + t.renameColumn('storeLinksTemp', 'storeLinks'); + }); +} + +async function revertStoreLinks(table: string, knex: any) { + await knex.schema.alterTable(table, (t: any) => { + t.json('storeLinksTemp'); + }); + if (knex.client.config.client === 'sqlite3') { + await knex(table).update({ + storeLinksTemp: knex.raw('"storeLinks"') + }) + } else { + await knex(table).update({ + storeLinksTemp: knex.raw('CAST("storeLinks" AS json)') + }) + } + await knex.schema.alterTable(table, (t: any) => { + t.dropColumn('storeLinks'); + }); + await knex.schema.alterTable(table, (t: any) => { + t.renameColumn('storeLinksTemp', 'storeLinks'); + }); +} + +export async function up(knex: Knex): Promise { + console.log("Knex client:", knex.client.config); + await castStoreLinks('songs', knex); + await castStoreLinks('albums', knex); + await castStoreLinks('artists', knex); +} + + +export async function down(knex: Knex): Promise { + await revertStoreLinks('songs', knex); + await revertStoreLinks('albums', knex); + await revertStoreLinks('artists', knex); +} + diff --git a/server/test/integration/flows/QueryFlow.js b/server/test/integration/flows/QueryFlow.js index b71afd3..c98d037 100644 --- a/server/test/integration/flows/QueryFlow.js +++ b/server/test/integration/flows/QueryFlow.js @@ -63,7 +63,7 @@ describe('POST /query with several songs and filters', () => { const song1 = { songId: 1, title: 'Song1', - storeLinks: [], + storeLinks: [ 'hello my', 'darling' ], artists: [ { artistId: 1, @@ -264,12 +264,43 @@ describe('POST /query with several songs and filters', () => { }); } + async function checkStoreLinksLike(req) { + await req + .post('/query') + .send({ + "query": { + "prop": "songStoreLinks", + "propOperator": "LIKE", + "propOperand": 'llo m' + }, + 'offsetsLimits': { + 'songOffset': 0, + 'songLimit': 10, + }, + 'ordering': { + 'orderBy': { + 'type': 0, + }, + 'ascending': true + } + }) + .then((res) => { + expect(res).to.have.status(200); + expect(res.body).to.deep.equal({ + songs: [song1], + artists: [], + tags: [], + albums: [], + }); + }); + } + let agent = await init(); let req = agent.keepOpen(); try { await helpers.createArtist(req, { name: "Artist1" }, 200); await helpers.createArtist(req, { name: "Artist2" }, 200); - await helpers.createSong(req, { title: "Song1", artistIds: [1] }, 200); + await helpers.createSong(req, { title: "Song1", artistIds: [1], storeLinks: [ 'hello my', 'darling' ] }, 200); await helpers.createSong(req, { title: "Song2", artistIds: [1] }, 200); await helpers.createSong(req, { title: "Song3", artistIds: [2] }, 200); await checkAllSongs(req); @@ -277,6 +308,7 @@ describe('POST /query with several songs and filters', () => { await checkIdNotIn(req); await checkArtistIdIn(req); await checkOrRelation(req); + await checkStoreLinksLike(req); } finally { req.close(); agent.close();