Have reasonable batch linking working for YouTube.

editsong
Sander Vocke 5 years ago
parent 342c4f0579
commit 02f6ea1a84
  1. 19
      client/src/api.ts
  2. 100
      client/src/components/windows/manage_links/BatchLinkDialog.tsx
  3. 19
      client/src/lib/backend/queries.tsx
  4. 151
      client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx
  5. 32
      server/endpoints/Album.ts
  6. 13
      server/endpoints/Artist.ts
  7. 30
      server/endpoints/Song.ts

@ -159,9 +159,9 @@ export interface SongDetailsRequest { }
export interface SongDetailsResponse {
title: string,
storeLinks: string[],
artistIds: number[],
albumIds: number[],
tagIds: number[],
artists?: ArtistDetailsResponseWithId[],
albums?: AlbumDetailsResponseWithId[],
tags?: TagDetailsResponseWithId[],
}
export function checkSongDetailsRequest(req: any): boolean {
return true;
@ -172,7 +172,7 @@ export const ArtistDetailsEndpoint = '/artist/:id';
export interface ArtistDetailsRequest { }
export interface ArtistDetailsResponse {
name: string,
tagIds: number[],
tags?: TagDetailsResponseWithId[],
storeLinks: string[],
}
export function checkArtistDetailsRequest(req: any): boolean {
@ -244,9 +244,9 @@ export const AlbumDetailsEndpoint = '/album/:id';
export interface AlbumDetailsRequest { }
export interface AlbumDetailsResponse {
name: string;
tagIds: number[];
artistIds: number[];
songIds: number[];
artists?: ArtistDetailsResponseWithId[],
songs?: SongDetailsResponseWithId[],
tags?: TagDetailsResponseWithId[],
storeLinks: string[];
}
export function checkAlbumDetailsRequest(req: any): boolean {
@ -469,3 +469,8 @@ export interface DeleteIntegrationResponse { }
export function checkDeleteIntegrationRequest(req: any): boolean {
return true;
}
export interface ArtistDetailsResponseWithId extends ArtistDetailsResponse { id: number }
export interface AlbumDetailsResponseWithId extends AlbumDetailsResponse { id: number }
export interface TagDetailsResponseWithId extends TagDetailsResponse { id: number }
export interface SongDetailsResponseWithId extends SongDetailsResponse { id: number }

@ -1,5 +1,5 @@
import React, { Dispatch, SetStateAction, useEffect, useState } from 'react';
import { Box, Button, Checkbox, createStyles, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, Divider, FormControlLabel, List, ListItem, ListItemIcon, ListItemText, makeStyles, MenuItem, Paper, Select, Theme, Typography } from "@material-ui/core";
import { Box, Button, Checkbox, createStyles, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, Divider, FormControlLabel, LinearProgress, List, ListItem, ListItemIcon, ListItemText, makeStyles, MenuItem, Paper, Select, Theme, Typography } from "@material-ui/core";
import StoreLinkIcon from '../../common/StoreLinkIcon';
import { $enum } from 'ts-enum-util';
import { IntegrationState, useIntegrations } from '../../../lib/integration/useIntegrations';
@ -11,6 +11,7 @@ import asyncPool from "tiny-async-pool";
import { getSong } from '../../../lib/backend/songs';
import { getAlbum } from '../../../lib/backend/albums';
import { getArtist } from '../../../lib/backend/artists';
import { modifyAlbum, modifyArtist, modifySong } from '../../../lib/saveChanges';
const useStyles = makeStyles((theme: Theme) =>
createStyles({
@ -24,6 +25,7 @@ enum BatchJobState {
Idle = 0,
Collecting,
Running,
Finished,
}
interface Task {
@ -99,11 +101,13 @@ async function doLinking(
console.log("Linking start!", toLink);
// Start the collecting phase.
setStatus({
setStatus((s: any) => {
return {
state: BatchJobState.Collecting,
numTasks: 0,
tasksSuccess: 0,
tasksFailed: 0,
}
});
console.log("Starting collection");
@ -127,17 +131,30 @@ async function doLinking(
console.log("Done collecting.", tasks)
// Start the linking phase.
setStatus((status: BatchJobStatus) => {
status.state = BatchJobState.Running;
status.numTasks = tasks.length;
console.log("Collected status:", status)
return status;
return {
...status,
state: BatchJobState.Running,
numTasks: tasks.length
}
});
let makeJob: (t: Task) => Promise<void> = (t: Task) => {
let integration = integrations.find((i: IntegrationState) => i.id === t.integrationId);
return (async () => {
let onSuccess = () => setStatus((s: BatchJobStatus) => { s.tasksSuccess += 1; return s; });
let onFail = () => setStatus((s: BatchJobStatus) => { s.tasksFailed += 1; return s; });
let onSuccess = () =>
setStatus((s: BatchJobStatus) => {
return {
...s,
tasksSuccess: s.tasksSuccess + 1,
}
});
let onFail = () =>
setStatus((s: BatchJobStatus) => {
return {
...s,
tasksFailed: s.tasksFailed + 1,
}
});
try {
if (integration === undefined) { return; }
console.log('integration search:', integration)
@ -154,18 +171,38 @@ async function doLinking(
[ItemType.Artist]: getArtist,
}
let queryFuncs: any = {
[ItemType.Song]: (s: any) => `${s.title}`,
[ItemType.Album]: (s: any) => `${s.name}`,
[ItemType.Song]: (s: any) => `${s.title}` +
`${s.artists && s.artists.length > 0 && ` ${s.artists[0].name}`}` +
`${s.albums && s.albums.length > 0 && ` ${s.albums[0].name}`}`,
[ItemType.Album]: (s: any) => `${s.name}` +
`${s.artists && s.artists.length > 0 && ` ${s.artists[0].name}`}`,
[ItemType.Artist]: (s: any) => `${s.name}`,
}
let query = queryFuncs[t.itemType](await getFuncs[t.itemType](t.itemId));
let modifyFuncs: any = {
[ItemType.Song]: modifySong,
[ItemType.Album]: modifyAlbum,
[ItemType.Artist]: modifyArtist,
}
let item = await getFuncs[t.itemType](t.itemId);
let query = queryFuncs[t.itemType](item);
let candidates = await searchFuncs[t.itemType](
query,
1,
);
var success = false;
if (candidates && candidates.length && candidates.length > 0 && candidates[0].url) {
await modifyFuncs[t.itemType](
t.itemId,
{
storeLinks: [...item.storeLinks, candidates[0].url]
}
)
success = true;
}
console.log(query, candidates);
if (candidates && candidates.length && candidates.length > 0) {
if (success) {
onSuccess();
} else {
onFail();
@ -178,13 +215,14 @@ async function doLinking(
})();
}
await asyncPool(4, tasks, makeJob);
await asyncPool(8, tasks, makeJob);
// Finalize.
setStatus((status: BatchJobStatus) => {
status.state = BatchJobState.Idle;
console.log("Done running:", status)
return status;
return {
...status,
state: BatchJobState.Finished,
}
});
}
@ -193,16 +231,31 @@ function ProgressDialog(props: {
onClose: () => void,
status: BatchJobStatus,
}) {
let donePercent = ((props.status.tasksFailed + props.status.tasksSuccess) / (props.status.numTasks || 1)) * 100;
return <Dialog
open={props.open}
onClose={props.onClose}
>
<DialogTitle>Batch linking in progress...</DialogTitle>
{props.status.state === BatchJobState.Collecting || props.status.state === BatchJobState.Running &&
<DialogTitle>Batch linking in progress...</DialogTitle>}
{props.status.state === BatchJobState.Finished &&
<DialogTitle>Batch linking finished</DialogTitle>}
<DialogContent>
{props.status.state === BatchJobState.Collecting || props.status.state === BatchJobState.Running &&
<DialogContentText>
Closing or refreshing this page will interrupt and abort the process.
</DialogContentText>
</DialogContentText>}
<Box minWidth="200px"><LinearProgress variant="determinate" color="secondary" value={donePercent} /></Box>
<Typography>
Found: {props.status.tasksSuccess}<br />
Failed: {props.status.tasksFailed}<br />
Total: {props.status.numTasks}<br />
</Typography>
</DialogContent>
{props.status.state === BatchJobState.Finished && <DialogActions>
<Button variant="contained" onClick={props.onClose}>Done</Button>
</DialogActions>}
</Dialog>
}
@ -380,8 +433,15 @@ export default function BatchLinkDialog(props: {
}}
/>
<ProgressDialog
open={jobStatus.state === BatchJobState.Collecting || jobStatus.state === BatchJobState.Running}
onClose={() => { }}
open={jobStatus.state === BatchJobState.Collecting || jobStatus.state === BatchJobState.Running || jobStatus.state === BatchJobState.Finished}
onClose={() => {
setJobStatus({
numTasks: 0,
tasksSuccess: 0,
tasksFailed: 0,
state: BatchJobState.Idle,
})
}}
status={jobStatus}
/>
</>

@ -24,17 +24,18 @@ export async function queryItems(
tags: number,
songs: number,
}> {
console.log("Types:", types);
var q: serverApi.QueryRequest = {
query: query ? toApiQuery(query) : {},
offsetsLimits: {
artistOffset: (serverApi.ItemType.Artist in types) ? (offset || 0) : undefined,
artistLimit: (serverApi.ItemType.Artist in types) ? (limit || -1) : undefined,
albumOffset: (serverApi.ItemType.Album in types) ? (offset || 0) : undefined,
albumLimit: (serverApi.ItemType.Album in types) ? (limit || -1) : undefined,
songOffset: (serverApi.ItemType.Song in types) ? (offset || 0) : undefined,
songLimit: (serverApi.ItemType.Song in types) ? (limit || -1) : undefined,
tagOffset: (serverApi.ItemType.Tag in types) ? (offset || 0) : undefined,
tagLimit: (serverApi.ItemType.Tag in types) ? (limit || -1) : undefined,
artistOffset: (types.includes(serverApi.ItemType.Artist)) ? (offset || 0) : undefined,
artistLimit: (types.includes(serverApi.ItemType.Artist)) ? (limit || -1) : undefined,
albumOffset: (types.includes(serverApi.ItemType.Album)) ? (offset || 0) : undefined,
albumLimit: (types.includes(serverApi.ItemType.Album)) ? (limit || -1) : undefined,
songOffset: (types.includes(serverApi.ItemType.Song)) ? (offset || 0) : undefined,
songLimit: (types.includes(serverApi.ItemType.Song)) ? (limit || -1) : undefined,
tagOffset: (types.includes(serverApi.ItemType.Tag)) ? (offset || 0) : undefined,
tagLimit: (types.includes(serverApi.ItemType.Tag)) ? (limit || -1) : undefined,
},
ordering: {
orderBy: {
@ -45,6 +46,8 @@ export async function queryItems(
responseType: responseType,
};
console.log(q);
const requestOpts = {
method: 'POST',
headers: { 'Content-Type': 'application/json' },

@ -35,7 +35,7 @@ export function extractInitialData(text: string): any | undefined {
export function parseSongs(initialData: any): IntegrationSong[] {
try {
var songMusicResponsiveListItemRenderers: any[] = [];
var musicResponsiveListItemRenderers: any[] = [];
// Scrape for any "Song"-type items.
initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
@ -45,13 +45,13 @@ export function parseSongs(initialData: any): IntegrationSong[] {
cc.musicResponsiveListItemRenderer.flexColumns &&
cc.musicResponsiveListItemRenderer.flexColumns[1]
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") {
songMusicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
musicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
}
})
}
})
return songMusicResponsiveListItemRenderers.map((s: any) => {
return musicResponsiveListItemRenderers.map((s: any) => {
let videoId = s.doubleTapCommand.watchEndpoint.videoId;
let columns = s.flexColumns;
@ -94,6 +94,82 @@ export function parseSongs(initialData: any): IntegrationSong[] {
}
}
export function parseArtists(initialData: any): IntegrationSong[] {
try {
var musicResponsiveListItemRenderers: any[] = [];
// Scrape for any "Artist"-type items.
initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
if (c.musicShelfRenderer) {
c.musicShelfRenderer.contents.forEach((cc: any) => {
if (cc.musicResponsiveListItemRenderer &&
cc.musicResponsiveListItemRenderer.flexColumns &&
cc.musicResponsiveListItemRenderer.flexColumns[1]
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Artist") {
musicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
}
})
}
})
return musicResponsiveListItemRenderers.map((s: any) => {
let browseId = s.navigationEndpoint.browseEndpoint.browseId;
let columns = s.flexColumns;
if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Artist") {
throw new Error('artist item doesnt match scraper expectation');
}
let name = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text;
return {
name: name,
url: `https://music.youtube.com/browse/${browseId}`,
}
})
} catch (e) {
console.log("Error parsing artists:", e.message);
return [];
}
}
export function parseAlbums(initialData: any): IntegrationSong[] {
try {
var musicResponsiveListItemRenderers: any[] = [];
// Scrape for any "Artist"-type items.
initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
if (c.musicShelfRenderer) {
c.musicShelfRenderer.contents.forEach((cc: any) => {
if (cc.musicResponsiveListItemRenderer &&
cc.musicResponsiveListItemRenderer.flexColumns &&
["Album", "Single"].includes(cc.musicResponsiveListItemRenderer.flexColumns[1]
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text)) {
musicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
}
})
}
})
return musicResponsiveListItemRenderers.map((s: any) => {
let browseId = s.navigationEndpoint.browseEndpoint.browseId;
let columns = s.flexColumns;
if (!["Album", "Single"].includes(columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text)) {
throw new Error('album item doesnt match scraper expectation');
}
let name = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text;
return {
name: name,
url: `https://music.youtube.com/browse/${browseId}`,
}
})
} catch (e) {
console.log("Error parsing albums:", e.message);
return [];
}
}
export default class YoutubeMusicWebScraper extends Integration {
integrationId: number;
@ -127,8 +203,6 @@ export default class YoutubeMusicWebScraper extends Integration {
let text = await response.text();
let songs = parseSongs(extractInitialData(text));
console.log("Found songs", songs);
if (!Array.isArray(songs) || songs.length === 0 || songs[0].title !== "No One Knows") {
throw new Error("Test failed; No One Knows was not correctly identified.");
}
@ -142,61 +216,20 @@ export default class YoutubeMusicWebScraper extends Integration {
let text = await response.text();
return parseSongs(extractInitialData(text));
}
async searchAlbum(query: string, limit: number): Promise<IntegrationAlbum[]> { return []; }
async searchArtist(query: string, limit: number): Promise<IntegrationArtist[]> { return []; }
async searchAlbum(query: string, limit: number): Promise<IntegrationAlbum[]> {
const response = await fetch(
(process.env.REACT_APP_BACKEND || "") +
`/integrations/${this.integrationId}/search?q=${encodeURIComponent(query)}`);
async search(query: string, type: SearchType, limit: number):
Promise<IntegrationSong[] | IntegrationAlbum[] | IntegrationArtist[]> {
let text = await response.text();
return parseAlbums(extractInitialData(text));
}
async searchArtist(query: string, limit: number): Promise<IntegrationArtist[]> {
const response = await fetch(
(process.env.REACT_APP_BACKEND || "") +
`/integrations/${this.integrationId}/search?q=${encodeURIComponent(query)}`);
return [];
// const response = await fetch(
// (process.env.REACT_APP_BACKEND || "") +
// `/integrations/${this.integrationId}/v1/search?q=${encodeURIComponent(query)}&type=${type}&limit=${limit}`);
// if (!response.ok) {
// throw new Error("Spotify Client Credentials search failed: " + JSON.stringify(response));
// }
// let json = await response.json();
// console.log("Response:", json);
// switch(type) {
// case SearchType.Song: {
// return json.tracks.items.map((r: any): IntegrationSong => {
// return {
// title: r.name,
// url: r.external_urls.spotify,
// artist: {
// name: r.artists && r.artists[0].name,
// url: r.artists && r.artists[0].external_urls.spotify,
// },
// album: {
// name: r.album && r.album.name,
// url: r.album && r.album.external_urls.spotify,
// }
// }
// })
// }
// case SearchType.Artist: {
// return json.artists.items.map((r: any): IntegrationArtist => {
// return {
// name: r.name,
// url: r.external_urls.spotify,
// }
// })
// }
// case SearchType.Album: {
// return json.albums.items.map((r: any): IntegrationAlbum => {
// return {
// name: r.name,
// url: r.external_urls.spotify,
// artist: {
// name: r.artists[0].name,
// url: r.artists[0].external_urls.spotify,
// },
// }
// })
// }
let text = await response.text();
return parseArtists(extractInitialData(text));
}
}

@ -17,24 +17,36 @@ export const GetAlbum: EndpointHandler = async (req: any, res: any, knex: Knex)
try {
// Start transfers for songs, tags and artists.
// Also request the album itself.
const tagIdsPromise = knex.select('tagId')
const tagsPromise: Promise<api.TagDetailsResponseWithId[]> = knex.select('tagId')
.from('albums_tags')
.where({ 'albumId': req.params.id })
.then((tags: any) => {
return tags.map((tag: any) => tag['tagId'])
});
const songIdsPromise = knex.select('songId')
})
.then((ids: number[]) => knex.select(['id', 'name', 'parentId'])
.from('tags')
.whereIn('id', ids));
const songsPromise: Promise<api.SongDetailsResponseWithId[]> = knex.select('songId')
.from('songs_albums')
.where({ 'albumId': req.params.id })
.then((songs: any) => {
return songs.map((song: any) => song['songId'])
});
const artistIdsPromise = knex.select('artistId')
})
.then((ids: number[]) => knex.select(['id', 'title', 'storeLinks'])
.from('songs')
.whereIn('id', ids));
const artistsPromise = knex.select('artistId')
.from('artists_albums')
.where({ 'albumId': req.params.id })
.then((artists: any) => {
return artists.map((artist: any) => artist['artistId'])
});
})
.then((ids: number[]) => knex.select(['id', 'name', 'storeLinks'])
.from('artists')
.whereIn('id', ids));
const albumPromise = knex.select('name', 'storeLinks')
.from('albums')
.where({ 'user': userId })
@ -43,15 +55,15 @@ export const GetAlbum: EndpointHandler = async (req: any, res: any, knex: Knex)
// Wait for the requests to finish.
const [album, tags, songs, artists] =
await Promise.all([albumPromise, tagIdsPromise, songIdsPromise, artistIdsPromise]);
await Promise.all([albumPromise, tagsPromise, songsPromise, artistsPromise]);
// Respond to the request.
if (album) {
const response: api.AlbumDetailsResponse = {
name: album['name'],
artistIds: artists,
tagIds: tags,
songIds: songs,
artists: artists,
tags: tags,
songs: songs,
storeLinks: asJson(album['storeLinks']),
};
await res.send(response);

@ -15,10 +15,17 @@ export const GetArtist: EndpointHandler = async (req: any, res: any, knex: Knex)
const { id: userId } = req.user;
try {
const tagIds = Array.from(new Set((await knex.select('tagId')
const tags: api.TagDetailsResponseWithId[] = await knex.select('tagId')
.from('artists_tags')
.where({ 'artistId': req.params.id })
).map((tag: any) => tag['tagId'])));
.then((ts: any) => {
return Array.from(new Set(
ts.map((tag: any) => tag['tagId'])
)) as number[];
})
.then((ids: number[]) => knex.select(['id', 'name', 'parentId'])
.from('tags')
.whereIn('id', ids));
const results = await knex.select(['id', 'name', 'storeLinks'])
.from('artists')
@ -28,7 +35,7 @@ export const GetArtist: EndpointHandler = async (req: any, res: any, knex: Knex)
if (results[0]) {
const response: api.ArtistDetailsResponse = {
name: results[0].name,
tagIds: tagIds,
tags: tags,
storeLinks: asJson(results[0].storeLinks),
}
await res.send(response);

@ -130,32 +130,42 @@ export const GetSong: EndpointHandler = async (req: any, res: any, knex: Knex) =
const { id: userId } = req.user;
try {
const tagIdsPromise: Promise<number[]> = knex.select('tagId')
const tagsPromise: Promise<api.TagDetailsResponseWithId[]> = knex.select('tagId')
.from('songs_tags')
.where({ 'songId': req.params.id })
.then((ts: any) => {
return Array.from(new Set(
ts.map((tag: any) => tag['tagId'])
));
)) as number[];
})
.then((ids: number[]) => knex.select(['id', 'name', 'parentId'])
.from('tags')
.whereIn('id', ids))
const albumIdsPromise: Promise<number[]> = knex.select('albumId')
const albumsPromise: Promise<api.AlbumDetailsResponseWithId[]> = knex.select('albumId')
.from('songs_albums')
.where({ 'songId': req.params.id })
.then((as: any) => {
return Array.from(new Set(
as.map((album: any) => album['albumId'])
));
)) as number[];
})
.then((ids: number[]) => knex.select(['id', 'name', 'storeLinks'])
.from('albums')
.whereIn('id', ids))
const artistIdsPromise: Promise<number[]> = knex.select('artistId')
const artistsPromise: Promise<api.ArtistDetailsResponseWithId[]> = knex.select('artistId')
.from('songs_artists')
.where({ 'songId': req.params.id })
.then((as: any) => {
return Array.from(new Set(
as.map((artist: any) => artist['artistId'])
));
)) as number[];
})
.then((ids: number[]) => knex.select(['id', 'name', 'storeLinks'])
.from('albums')
.whereIn('id', ids))
const songPromise = await knex.select(['id', 'title', 'storeLinks'])
.from('songs')
.where({ 'user': userId })
@ -163,14 +173,14 @@ export const GetSong: EndpointHandler = async (req: any, res: any, knex: Knex) =
.then((ss: any) => ss[0])
const [tags, albums, artists, song] =
await Promise.all([tagIdsPromise, albumIdsPromise, artistIdsPromise, songPromise]);
await Promise.all([tagsPromise, albumsPromise, artistsPromise, songPromise]);
if (song) {
const response: api.SongDetailsResponse = {
title: song.title,
tagIds: tags,
artistIds: artists,
albumIds: albums,
tags: tags,
artists: artists,
albums: albums,
storeLinks: asJson(song.storeLinks),
}
await res.send(response);

Loading…
Cancel
Save