sander
/
MuDBase
1
0
Fork 0
Code Issues 17 Pull Requests Projects 6 Releases Wiki Activity

Got Youtube Music scraper working again.
continuous-integration/drone/push Build was killed
Details

Browse Source
editsong
Sander Vocke 5 years ago
parent af5a742d7c
commit 600a8ced5d
4 changed files with 117 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 13
      .vscode/launch.json
  2. 5
      client/src/components/windows/settings/IntegrationSettings.tsx
  3. 132
      client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx
  4. 6
      server/integrations/integrations.ts

13
.vscode/launch.json vendored
Unescape View File

@ -19,6 +19,19 @@
"console": "integratedTerminal",
"cwd": "${workspaceFolder}/server",
"internalConsoleOptions": "neverOpen"
},
{
"type": "node",
"request": "launch",
"name": "Development Server with SQLite @ dev.sqlite3",
"env": {
"API": "/api",
},
"program": "node_modules/.bin/nodemon",
"args": [ "server.ts" ],
"console": "integratedTerminal",
"cwd": "${workspaceFolder}/server",
"internalConsoleOptions": "neverOpen"
}
]
}

5
client/src/components/windows/settings/IntegrationSettings.tsx
Unescape View File

@ -195,6 +195,11 @@ let EditorWithTest = (props: any) => {
<Alert severity="success">Integration is active.</Alert>
)
})
.catch((e: any) => {
setTestFlashMessage(
<Alert severity="error">Failed test: {e.message}</Alert>
)
})
}}
flashMessage={testFlashMessage}
showTestButton={true}

132
client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx
Unescape View File

@ -2,6 +2,8 @@ import React from 'react';
import Integration, { IntegrationFeature, IntegrationAlbum, IntegrationArtist, IntegrationTrack } from '../Integration';
import StoreLinkIcon from '../../../components/common/StoreLinkIcon';
import { IntegrationWith } from '../../../api/api';
import { runInNewContext } from 'vm';
import { TextRotateVertical } from '@material-ui/icons';
enum SearchType {
Track = 'track',
@ -19,82 +21,134 @@ export function extractInitialData(text: string): any | undefined {
// });
//
// the THIS part.
//
// Another variant was found in the field, where there was also additional encoding involved:
//
// initialData.push({
// path: '\/search',
// params: JSON.parse('\x7b\x22query\x22:\x22something\x22\x7d')
// data: 'THIS2'
// })
// , where THIS2 was a string which also contained escape characters like \x7b and \x22.
// Get the whole line containing the data part.
// Handle the 1st case.
let pattern = /initialData\.push\({[\n\r\s]*path:.*[\n\r\s]+params:\s*{\s*['"]query['"].*[\n\r\s]+data:\s*['"](.*)['"]\s*[\n\r]/
let m = text.match(pattern);
let dataline = Array.isArray(m) && m.length >= 2 ? m[1] : undefined;
if (!dataline) { return undefined; }
let dataline1 = Array.isArray(m) && m.length >= 2 ? m[1] : undefined;
// Now parse the data line.
let dataline_clean = dataline.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"')
let dataline1_clean = dataline1 ? dataline1.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"') : undefined;
let json1 = dataline1_clean ? JSON.parse(dataline1_clean) : undefined;
// Handle the 2nd case.
let m2 = text.match(/params:[\s]*JSON\.parse\('([^']*)'\),[\n\r\s]*data:[\s]*'([^']*)'/g);
let json2: any = undefined;
if (Array.isArray(m2)) {
m2.forEach((match: string) => {
let decode = (s: string) => {
var r = /\\x([\d\w]{2})/gi;
let res = s.replace(r, function (match, grp) {
return String.fromCharCode(parseInt(grp, 16));
});
return unescape(res);
}
let paramsline: string = decode((match.match(/params:[\s]*JSON\.parse\('([^']*)'/) as string[])[1]);
if (!('query' in JSON.parse(paramsline))) {
return;
}
let dataline2: string = decode((match.match(/data:[\s]*'([^']*)'/) as string[])[1]);
json2 = JSON.parse(dataline2);
})
}
let json = JSON.parse(dataline_clean);
return json;
// Return either one that worked.
let result = json1 || json2;
console.log("initial data:", result);
return result;
}
export function parseTracks(initialData: any): IntegrationTrack[] {
try {
var musicResponsiveListItemRenderers: any[] = [];
// Scrape for any "Track"-type items.
// Scrape for any "Song"-type items.
initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
if (c.musicShelfRenderer) {
c.musicShelfRenderer.contents.forEach((cc: any) => {
if (cc.musicResponsiveListItemRenderer &&
cc.musicResponsiveListItemRenderer.flexColumns &&
cc.musicResponsiveListItemRenderer.flexColumns[1]
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Track") {
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") {
musicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
}
})
}
})
return musicResponsiveListItemRenderers.map((s: any) => {
let videoId = s.doubleTapCommand.watchEndpoint.videoId;
let columns = s.flexColumns;
if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Track") {
throw new Error('song item doesnt match scraper expectation');
}
let title = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text;
let artists = columns[2].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
return 'navigationEndpoint' in run;
}).map((run: any) => {
let id = run.navigationEndpoint.browseEndpoint.browseId;
return {
url: `https://music.youtube.com/browse/${id}`,
name: run.text,
}
});
console.log("Found song itemrenderers:", musicResponsiveListItemRenderers);
let albums = columns[3].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
return 'navigationEndpoint' in run;
}).map((run: any) => {
let id = run.navigationEndpoint.browseEndpoint.browseId;
return {
url: `https://music.youtube.com/browse/${id}`,
name: run.text,
artist: artists[0],
return musicResponsiveListItemRenderers.map((s: any) => {
// There are some options that were encountered in the field.
// let videoId: string | undefined = undefined;
// if('doubleTapCommand' in s) s = s || s.doubleTapCommand.watchEndpoint.videoId;
// if('playlistItemData' in s) s = s || s.playlistItemData.videoId;
let runs: any[] = [];
// Gather all 'runs' fields together from all columns.
s.flexColumns.forEach((column: any) => {
runs.push(...column.musicResponsiveListItemFlexColumnRenderer.text.runs);
})
// Find the runs that hold the title, artist or album.
let title: string | undefined = undefined;
let album: IntegrationAlbum = {};
let artist: IntegrationArtist = {};
let videoId: string | undefined = undefined;
runs.forEach((run: any) => {
if ('navigationEndpoint' in run &&
'watchEndpoint' in run.navigationEndpoint &&
'videoId' in run.navigationEndpoint.watchEndpoint) {
videoId = run.navigationEndpoint.watchEndpoint.videoId;
title = run.text;
} else if ('navigationEndpoint' in run &&
'browseEndpoint' in run.navigationEndpoint &&
'browseEndpointContextSupportedConfigs' in run.navigationEndpoint.browseEndpoint &&
'browseEndpointContextMusicConfig' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs &&
'pageType' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig &&
run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig.pageType === 'MUSIC_PAGE_TYPE_ALBUM') {
album = {
url: `https://music.youtube.com/browse/${run.navigationEndpoint.browseEndpoint.browseId}`,
name: run.text,
}
} else if ('navigationEndpoint' in run &&
'browseEndpoint' in run.navigationEndpoint &&
'browseEndpointContextSupportedConfigs' in run.navigationEndpoint.browseEndpoint &&
'browseEndpointContextMusicConfig' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs &&
'pageType' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig &&
run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig.pageType === 'MUSIC_PAGE_TYPE_ARTIST') {
artist = {
url: `https://music.youtube.com/browse/${run.navigationEndpoint.browseEndpoint.browseId}`,
name: run.text,
}
}
});
if(album.name && artist.name) {
album.artist = artist;
}
return {
title: title,
url: `https://music.youtube.com/watch?v=${videoId}`,
artist: artists[0],
album: albums[0],
artist: artist,
album: album,
}
})
} catch (e) {
console.log("Error parsing songs:", e.message);
console.log("Error parsing tracks:", e.message);
return [];
}
}
export function parseArtists(initialData: any): IntegrationTrack[] {
export function parseArtists(initialData: any): IntegrationArtist[] {
try {
var musicResponsiveListItemRenderers: any[] = [];

6
server/integrations/integrations.ts
Unescape View File

@ -45,6 +45,9 @@ export function createIntegrations(knex: Knex, apiBaseUrl: string) {
let replaced = path.replace(new RegExp(`${apiBaseUrl}/integrations/[0-9]+/`), '');
console.log("Rewrite URL:", path, replaced);
return replaced;
},
onProxyReq: (proxyReq: any, req: any, res: any) => {
console.log('--> ', req.method, req.path, '->', proxyReq.path);
}
});
@ -57,6 +60,9 @@ export function createIntegrations(knex: Knex, apiBaseUrl: string) {
let replaced = path.replace(new RegExp(`${apiBaseUrl}/integrations/[0-9]+/`), '');
console.log("Rewrite URL:", path, replaced);
return replaced;
},
onProxyReq: (proxyReq: any, req: any, res: any) => {
console.log('--> ', req.method, req.path, '->', proxyReq.path);
}
})

Write Preview
Loading…
Cancel
Save