Fix tests.

client/src/api.ts

@@ -94,12 +94,7 @@ export interface Ordering {
 export interface Query extends QueryElem { }
 export interface QueryRequest {
     query: Query,
-    songOffset: number,
+    offsetsLimits: OffsetsLimits,
-    songLimit: number,
-    artistOffset: number,
-    artistLimit: number,
-    tagOffset: number,
-    tagLimit: number,
     ordering: Ordering,
 }
 export interface QueryResponse {
@@ -107,6 +102,14 @@ export interface QueryResponse {
     artists: ArtistDetails[],
     tags: TagDetails[],
 }
+export interface OffsetsLimits {
+    songOffset?: number,
+    songLimit?: number,
+    artistOffset?: number,
+    artistLimit?: number,
+    tagOffset?: number,
+    tagLimit?: number,
+}
 export function checkQueryElem(elem: any): boolean {
     if (elem.childrenOperator && elem.children) {
         elem.children.forEach((child: any) => {
@@ -121,12 +124,8 @@ export function checkQueryElem(elem: any): boolean {
 }
 export function checkQueryRequest(req: any): boolean {
     return 'query' in req
-        && 'songOffset' in req
+        && 'offsetsLimits' in req
-        && 'songLimit' in req
+        && 'ordering' in req
-        && 'artistOffset' in req
-        && 'artistLimit' in req
-        && 'tagOffset' in req
-        && 'tagLimit' in req
         && checkQueryElem(req.query);
 }
 

scripts/gpm_retrieve/gpm_retrieve.py

@@ -4,15 +4,14 @@ from gmusicapi import Mobileclient
 import argparse
 import sys
 import requests
+import json
 
 creds_path=sys.path[0] + '/mobileclient.cred'
 
 def authenticate(api):
     creds = api.perform_oauth(storage_filepath=creds_path, open_browser=False)
 
-def transferLibrary(gpm_api, mudbase_api):
+def uploadLibrary(mudbase_api, songs):    
-    songs = gpm_api.get_all_songs()
-    
     # Determine all unique artists
     artists = sorted(set([song['artist'] for song in songs if 'artist' in song]))
 
@@ -20,10 +19,10 @@ def transferLibrary(gpm_api, mudbase_api):
     genres = sorted(set([song['genre'] for song in songs if 'genre' in song]))
 
     # Store the artist index of each song
-    songArtistIdxs = [ artists.index(song['artist']) for song in songs ]
+    songArtistIdxs = [ artists.index(song['artist']) for song in songs if 'artist' in song ]
 
     # Store the genre index of each song
-    songArtistIdxs = [ genres.index(song['genre']) for song in songs ]
+    songGenreIdxs = [ genres.index(song['genre']) for song in songs if 'genre' in song ]
     
     # Determine all unique albums per artist
     artistAlbums = [ sorted(set([ song['album'] for song in songs if song['artist'] == artist ])) for artist in artists ]
@@ -70,28 +69,77 @@ def transferLibrary(gpm_api, mudbase_api):
     def getSongStoreIds(song):
         if 'storeId' in song:
             return [ song['storeId'] ]
-        return [];
+        return []
     for song in songs:
         artistMudbaseId = artistMudbaseIds[ artists.index(song['artist']) ]
+        tagIds = [ gpmTagIdResponse['id'] ]
+        if 'genre' in song:
             genreMudbaseId = genreMudbaseIds[ genres.index(song['genre']) ]
+            tagIds.append(genreMudbaseId)
+
         response = requests.post(mudbase_api + '/song', json = {
             'title': song['title'],
             'artistIds': [ artistMudbaseId ],
-            'tagIds' : [ genreMudbaseId, gpmTagIdResponse['id'] ],
+            'tagIds' : tagIds,
             'storeLinks': [ 'https://play.google.com/music/m/' + id for id in getSongStoreIds(song) ],
         }).json()
         print(f"Created song \"{song['title']}\" with artist ID {artistMudbaseId}, response: {response}")
 
+def getData(api):
+    return {
+        "songs": api.get_all_songs(),
+        "playlists": api.get_all_user_playlist_contents()
+    }
+
+
+def getSongs(data):
+    # Get songs from library
+    songs = [] #data['songs']
+
+    # Append songs from playlists
+    for playlist in data['playlists']:
+        for track in playlist['tracks']:
+            if 'track' in track:
+                songs.append(track['track'])
+
+    # Uniquify by using a dict. After all, same song may appear in
+    # multiple playlists.
+    sI = lambda song: song['artist'] + '-' + song['title'] if 'artist' in song and 'title' in song else 'z'
+    return list(dict((sI(song), song) for song in songs).values())
+
 api = Mobileclient()
 
 parser = argparse.ArgumentParser(description="Import Google Music library into MudBase.")
 parser.add_argument('--authenticate', help="Generate credentials for authentication", action="store_true")
-parser.add_argument('mudbase_api', help="Address for the Mudbase back-end API")
+parser.add_argument('--store-to', help="Store GPM library to JSON for later upload", action='store', dest='store_to')
+parser.add_argument('--load-from', help="Load GPM library from JSON for upload", action='store', dest='load_from')
+parser.add_argument('--mudbase_api', help="Address for the Mudbase back-end API to upload to", action='store', dest='mudbase_api')
 
 args = parser.parse_args()
 
 if args.authenticate:
     authenticate(api)
-else:
+
+data = None
+
+# Determine whether we need to log in to GPM and get songs
+if args.store_to or (not args.load_from and args.mudbase_api):
+    api.oauth_login(Mobileclient.FROM_MAC_ADDRESS, oauth_credentials=creds_path)
+    data = getData(api)
+
+# Determine whether to save to a file
+if args.store_to:
+    with open(args.store_to, 'w') as outfile:
+        json.dump(data, outfile, sort_keys=True, indent=2)
+
+# Determine whether to load from a file
+if args.load_from:
+    with open(args.load_from, 'r') as f:
+        data = json.load(f)
+
+songs = getSongs(data)
+print(f"Found {len(songs)} songs.")
+
+if args.mudbase_api:
     api.oauth_login(Mobileclient.FROM_MAC_ADDRESS, oauth_credentials=creds_path)
-    transferLibrary(api, args.mudbase_api)
+    uploadLibrary(args.mudbase_api, songs)

server/endpoints/QueryEndpointHandler.ts

@@ -99,7 +99,14 @@ export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any)
     const reqObject: api.QueryRequest = req.body;
 
     try {
-        const songs = (reqObject.songLimit > 0) && await models.Song.findAll({
+        const songLimit = reqObject.offsetsLimits.songLimit;
+        const songOffset = reqObject.offsetsLimits.songOffset;
+        const tagLimit = reqObject.offsetsLimits.tagLimit;
+        const tagOffset = reqObject.offsetsLimits.tagOffset;
+        const artistLimit = reqObject.offsetsLimits.artistLimit;
+        const artistOffset = reqObject.offsetsLimits.artistOffset;
+
+        const songs = (songLimit && songLimit > 0) && await models.Song.findAll({
             // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
             // Custom pagination is implemented before responding.
             where: getSequelizeWhere(reqObject.query, QueryType.Song),
@@ -108,7 +115,7 @@ export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any)
             //limit: reqObject.limit,
             //offset: reqObject.offset,
         })
-        const artists = (reqObject.artistLimit > 0) && await models.Artist.findAll({
+        const artists = (artistLimit && artistLimit > 0) && await models.Artist.findAll({
             // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
             // Custom pagination is implemented before responding.
             where: getSequelizeWhere(reqObject.query, QueryType.Artist),
@@ -117,7 +124,7 @@ export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any)
             //limit: reqObject.limit,
             //offset: reqObject.offset,
         })
-        const tags = (reqObject.tagLimit > 0) && await models.Tag.findAll({
+        const tags = (tagLimit && tagLimit > 0) && await models.Tag.findAll({
             // NOTE: have to disable limit and offset because of bug: https://github.com/sequelize/sequelize/issues/11938.
             // Custom pagination is implemented before responding.
             where: getSequelizeWhere(reqObject.query, QueryType.Tag),
@@ -128,7 +135,7 @@ export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any)
         })
 
         const response: api.QueryResponse = {
-            songs: (reqObject.songLimit <= 0) ? [] : await Promise.all(songs.map(async (song: any) => {
+            songs: ((songLimit || -1) <= 0) ? [] : await Promise.all(songs.map(async (song: any) => {
                 const artists = song.getArtists();
                 const tags = song.getTags();
                 const rankings = song.getRankings();
@@ -161,20 +168,20 @@ export const QueryEndpointHandler: EndpointHandler = async (req: any, res: any)
                         }
                     })
                 };
-            }).slice(reqObject.songOffset, reqObject.songOffset + reqObject.songLimit)),
+            }).slice(songOffset || 0, (songOffset || 0) + (songLimit || 10))),
             // TODO: custom pagination due to bug mentioned above
-            artists: (reqObject.artistLimit <= 0) ? [] : await Promise.all(artists.map(async (artist: any) => {
+            artists: ((artistLimit || -1) <= 0) ? [] : await Promise.all(artists.map(async (artist: any) => {
                 return <api.ArtistDetails>{
                     artistId: artist.id,
                     name: artist.name,
                 };
-            }).slice(reqObject.artistOffset, reqObject.artistOffset + reqObject.artistLimit)),
+            }).slice(artistOffset || 0, (artistOffset || 0) + (artistLimit || 10))),
-            tags: (reqObject.tagLimit <= 0) ? [] : await Promise.all(tags.map(async (tag: any) => {
+            tags: ((tagLimit || -1) <= 0) ? [] : await Promise.all(tags.map(async (tag: any) => {
                 return <api.TagDetails>{
                     tagId: tag.id,
                     name: tag.name,
                 };
-            }).slice(reqObject.tagOffset, reqObject.tagOffset + reqObject.tagLimit)),
+            }).slice(tagOffset || 0,  (tagOffset || 0) + (tagLimit || 10))),
         };
         res.send(response);
     } catch (e) {

server/test/integration/flows/QueryFlow.js

@@ -0,0 +1,162 @@
+const chai = require('chai');
+const chaiHttp = require('chai-http');
+const express = require('express');
+const models = require('../../../models');
+import { SetupApp } from '../../../app';
+import { expect } from 'chai';
+import * as helpers from './helpers';
+
+async function init() {
+    chai.use(chaiHttp);
+    const app = express();
+    SetupApp(app);
+    await models.sequelize.sync({ force: true });
+    return app;
+}
+
+describe('POST /query with no songs', () => {
+    it('should give empty list', done => {
+        init().then((app) => {
+            chai
+                .request(app)
+                .post('/query')
+                .send({
+                    'query': {},
+                    'offsetsLimits': {
+                        'songOffset': 0,
+                        'songLimit': 10,
+                    },
+                    'ordering': {
+                        'orderBy': {
+                            'type': 0,
+                        },
+                        'ascending': true
+                    }
+                })
+                .then((res) => {
+                    expect(res).to.have.status(200);
+                    expect(res.body).to.deep.equal({
+                        songs: [],
+                        tags: [],
+                        artists: [],
+                    });
+                    done();
+                });
+        })
+    });
+});
+
+describe('POST /query with several songs and filters', () => {
+    it('should give all correct results', done => {
+        init().then((app) => {
+            async function checkAllSongs(req) {
+                await req
+                    .post('/query')
+                    .send({ "query": {} })
+                    .then((res) => {
+                        expect(res).to.have.status(200);
+                        expect(res.body).to.deep.equal({
+                            ids: [1, 2, 3]
+                        });
+                    });
+            }
+
+            async function checkIdIn(req) {
+                await req
+                    .post('/query')
+                    .send({
+                        "query": {
+                            "prop": "id",
+                            "propOperator": "IN",
+                            "propOperand": [1, 3, 5]
+                        }
+                    })
+                    .then((res) => {
+                        expect(res).to.have.status(200);
+                        expect(res.body).to.deep.equal({
+                            ids: [1, 3]
+                        });
+                    });
+            }
+
+            async function checkIdNotIn(req) {
+                await req
+                    .post('/query')
+                    .send({
+                        "query": {
+                            "prop": "id",
+                            "propOperator": "NOTIN",
+                            "propOperand": [1, 3, 5]
+                        }
+                    })
+                    .then((res) => {
+                        expect(res).to.have.status(200);
+                        expect(res.body).to.deep.equal({
+                            ids: [2]
+                        });
+                    });
+            }
+
+            async function checkArtistIdIn(req) {
+                await req
+                    .post('/query')
+                    .send({
+                        "query": {
+                            "prop": "artistIds",
+                            "propOperator": "IN",
+                            "propOperand": [1]
+                        }
+                    })
+                    .then((res) => {
+                        expect(res).to.have.status(200);
+                        expect(res.body).to.deep.equal({
+                            ids: [1, 2]
+                        });
+                    });
+            }
+
+            async function checkOrRelation(req) {
+                await req
+                    .post('/query')
+                    .send({
+                        "query": {
+                            "childrenOperator": "OR",
+                            "children": [
+                                {
+                                    "prop": "artistIds",
+                                    "propOperator": "IN",
+                                    "propOperand": [2]
+                                },
+                                {
+                                    "prop": "id",
+                                    "propOperator": "EQ",
+                                    "propOperand": 1
+                                }
+                            ]
+                        }
+                    })
+                    .then((res) => {
+                        expect(res).to.have.status(200);
+                        expect(res.body).to.deep.equal({
+                            ids: [1, 3]
+                        });
+                    });
+            }
+
+            var req = chai.request(app).keepOpen();
+
+            helpers.createArtist(req, { name: "Artist1" }, 200)
+                .then(() => helpers.createArtist(req, { name: "Artist2" }, 200))
+                .then(() => helpers.createSong(req, { title: "Song1", artistIds: [1] }, 200))
+                .then(() => helpers.createSong(req, { title: "Song2", artistIds: [1] }, 200))
+                .then(() => helpers.createSong(req, { title: "Song3", artistIds: [2] }, 200))
+                .then(() => checkAllSongs(req))
+                .then(() => checkIdIn(req))
+                .then(() => checkIdNotIn(req))
+                .then(() => checkArtistIdIn(req))
+                .then(() => checkOrRelation(req))
+                .then(req.close)
+                .then(done)
+        })
+    });
+});

server/test/integration/flows/SongFlow.js

@@ -104,142 +104,6 @@ describe('POST /song with an existent and a nonexistent artist Id', () => {
     });
 });
 
-
-describe('POST /song/query with no songs', () => {
-    it('should give empty list', done => {
-        init().then((app) => {
-            chai
-                .request(app)
-                .post('/song/query')
-                .send({
-                    'query': {}
-                })
-                .then((res) => {
-                    expect(res).to.have.status(200);
-                    expect(res.body).to.deep.equal({
-                        ids: []
-                    });
-                    done();
-                });
-        })
-    });
-});
-
-describe('POST /song/query with several songs and filters', () => {
-    it('should give all correct results', done => {
-        init().then((app) => {
-            async function checkAllSongs(req) {
-                await req
-                    .post('/song/query')
-                    .send({ "query": {} })
-                    .then((res) => {
-                        expect(res).to.have.status(200);
-                        expect(res.body).to.deep.equal({
-                            ids: [1, 2, 3]
-                        });
-                    });
-            }
-
-            async function checkIdIn(req) {
-                await req
-                    .post('/song/query')
-                    .send({
-                        "query": {
-                            "prop": "id",
-                            "propOperator": "IN",
-                            "propOperand": [1, 3, 5]
-                        }
-                    })
-                    .then((res) => {
-                        expect(res).to.have.status(200);
-                        expect(res.body).to.deep.equal({
-                            ids: [1, 3]
-                        });
-                    });
-            }
-
-            async function checkIdNotIn(req) {
-                await req
-                    .post('/song/query')
-                    .send({
-                        "query": {
-                            "prop": "id",
-                            "propOperator": "NOTIN",
-                            "propOperand": [1, 3, 5]
-                        }
-                    })
-                    .then((res) => {
-                        expect(res).to.have.status(200);
-                        expect(res.body).to.deep.equal({
-                            ids: [2]
-                        });
-                    });
-            }
-
-            async function checkArtistIdIn(req) {
-                await req
-                    .post('/song/query')
-                    .send({
-                        "query": {
-                            "prop": "artistIds",
-                            "propOperator": "IN",
-                            "propOperand": [1]
-                        }
-                    })
-                    .then((res) => {
-                        expect(res).to.have.status(200);
-                        expect(res.body).to.deep.equal({
-                            ids: [1, 2]
-                        });
-                    });
-            }
-
-            async function checkOrRelation(req) {
-                await req
-                    .post('/song/query')
-                    .send({
-                        "query": {
-                            "childrenOperator": "OR",
-                            "children": [
-                                {
-                                    "prop": "artistIds",
-                                    "propOperator": "IN",
-                                    "propOperand": [2]
-                                },
-                                {
-                                    "prop": "id",
-                                    "propOperator": "EQ",
-                                    "propOperand": 1
-                                }
-                            ]
-                        }
-                    })
-                    .then((res) => {
-                        expect(res).to.have.status(200);
-                        expect(res.body).to.deep.equal({
-                            ids: [1, 3]
-                        });
-                    });
-            }
-
-            var req = chai.request(app).keepOpen();
-
-            helpers.createArtist(req, { name: "Artist1" }, 200)
-                .then(() => helpers.createArtist(req, { name: "Artist2" }, 200))
-                .then(() => helpers.createSong(req, { title: "Song1", artistIds: [1] }, 200))
-                .then(() => helpers.createSong(req, { title: "Song2", artistIds: [1] }, 200))
-                .then(() => helpers.createSong(req, { title: "Song3", artistIds: [2] }, 200))
-                .then(() => checkAllSongs(req))
-                .then(() => checkIdIn(req))
-                .then(() => checkIdNotIn(req))
-                .then(() => checkArtistIdIn(req))
-                .then(() => checkOrRelation(req))
-                .then(req.close)
-                .then(done)
-        })
-    });
-});
-
 describe('POST /song with tags', () => {
     it('should succeed', done => {
         init().then((app) => {

test

@@ -1 +0,0 @@
-a