diff --git a/.vscode/launch.json b/.vscode/launch.json index 56c4732..badd60b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,6 +19,19 @@ "console": "integratedTerminal", "cwd": "${workspaceFolder}/server", "internalConsoleOptions": "neverOpen" + }, + { + "type": "node", + "request": "launch", + "name": "Development Server with SQLite @ dev.sqlite3", + "env": { + "API": "/api", + }, + "program": "node_modules/.bin/nodemon", + "args": [ "server.ts" ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/server", + "internalConsoleOptions": "neverOpen" } ] } \ No newline at end of file diff --git a/client/src/components/windows/settings/IntegrationSettings.tsx b/client/src/components/windows/settings/IntegrationSettings.tsx index 250981d..c11ec45 100644 --- a/client/src/components/windows/settings/IntegrationSettings.tsx +++ b/client/src/components/windows/settings/IntegrationSettings.tsx @@ -195,6 +195,11 @@ let EditorWithTest = (props: any) => { Integration is active. ) }) + .catch((e: any) => { + setTestFlashMessage( + Failed test: {e.message} + ) + }) }} flashMessage={testFlashMessage} showTestButton={true} diff --git a/client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx b/client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx index 6bb7c2b..4184ddb 100644 --- a/client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx +++ b/client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx @@ -2,6 +2,8 @@ import React from 'react'; import Integration, { IntegrationFeature, IntegrationAlbum, IntegrationArtist, IntegrationTrack } from '../Integration'; import StoreLinkIcon from '../../../components/common/StoreLinkIcon'; import { IntegrationWith } from '../../../api/api'; +import { runInNewContext } from 'vm'; +import { TextRotateVertical } from '@material-ui/icons'; enum SearchType { Track = 'track', @@ -19,82 +21,134 @@ export function extractInitialData(text: string): any | undefined { // }); // // the THIS part. + // + // Another variant was found in the field, where there was also additional encoding involved: + // + // initialData.push({ + // path: '\/search', + // params: JSON.parse('\x7b\x22query\x22:\x22something\x22\x7d') + // data: 'THIS2' + // }) + // , where THIS2 was a string which also contained escape characters like \x7b and \x22. - // Get the whole line containing the data part. + // Handle the 1st case. let pattern = /initialData\.push\({[\n\r\s]*path:.*[\n\r\s]+params:\s*{\s*['"]query['"].*[\n\r\s]+data:\s*['"](.*)['"]\s*[\n\r]/ let m = text.match(pattern); - let dataline = Array.isArray(m) && m.length >= 2 ? m[1] : undefined; - if (!dataline) { return undefined; } - + let dataline1 = Array.isArray(m) && m.length >= 2 ? m[1] : undefined; // Now parse the data line. - let dataline_clean = dataline.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"') + let dataline1_clean = dataline1 ? dataline1.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"') : undefined; + let json1 = dataline1_clean ? JSON.parse(dataline1_clean) : undefined; + + // Handle the 2nd case. + let m2 = text.match(/params:[\s]*JSON\.parse\('([^']*)'\),[\n\r\s]*data:[\s]*'([^']*)'/g); + let json2: any = undefined; + if (Array.isArray(m2)) { + m2.forEach((match: string) => { + let decode = (s: string) => { + var r = /\\x([\d\w]{2})/gi; + let res = s.replace(r, function (match, grp) { + return String.fromCharCode(parseInt(grp, 16)); + }); + return unescape(res); + } + let paramsline: string = decode((match.match(/params:[\s]*JSON\.parse\('([^']*)'/) as string[])[1]); + if (!('query' in JSON.parse(paramsline))) { + return; + } + let dataline2: string = decode((match.match(/data:[\s]*'([^']*)'/) as string[])[1]); + json2 = JSON.parse(dataline2); + }) + } - let json = JSON.parse(dataline_clean); - return json; + // Return either one that worked. + let result = json1 || json2; + console.log("initial data:", result); + return result; } export function parseTracks(initialData: any): IntegrationTrack[] { try { var musicResponsiveListItemRenderers: any[] = []; - // Scrape for any "Track"-type items. + // Scrape for any "Song"-type items. initialData.contents.sectionListRenderer.contents.forEach((c: any) => { if (c.musicShelfRenderer) { c.musicShelfRenderer.contents.forEach((cc: any) => { if (cc.musicResponsiveListItemRenderer && cc.musicResponsiveListItemRenderer.flexColumns && cc.musicResponsiveListItemRenderer.flexColumns[1] - .musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Track") { + .musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") { musicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer); } }) } }) - return musicResponsiveListItemRenderers.map((s: any) => { - let videoId = s.doubleTapCommand.watchEndpoint.videoId; - let columns = s.flexColumns; - - if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Track") { - throw new Error('song item doesnt match scraper expectation'); - } - let title = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text; - - let artists = columns[2].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => { - return 'navigationEndpoint' in run; - }).map((run: any) => { - let id = run.navigationEndpoint.browseEndpoint.browseId; - return { - url: `https://music.youtube.com/browse/${id}`, - name: run.text, - } - }); + console.log("Found song itemrenderers:", musicResponsiveListItemRenderers); - let albums = columns[3].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => { - return 'navigationEndpoint' in run; - }).map((run: any) => { - let id = run.navigationEndpoint.browseEndpoint.browseId; - return { - url: `https://music.youtube.com/browse/${id}`, - name: run.text, - artist: artists[0], + return musicResponsiveListItemRenderers.map((s: any) => { + // There are some options that were encountered in the field. + // let videoId: string | undefined = undefined; + // if('doubleTapCommand' in s) s = s || s.doubleTapCommand.watchEndpoint.videoId; + // if('playlistItemData' in s) s = s || s.playlistItemData.videoId; + + let runs: any[] = []; + // Gather all 'runs' fields together from all columns. + s.flexColumns.forEach((column: any) => { + runs.push(...column.musicResponsiveListItemFlexColumnRenderer.text.runs); + }) + + // Find the runs that hold the title, artist or album. + let title: string | undefined = undefined; + let album: IntegrationAlbum = {}; + let artist: IntegrationArtist = {}; + let videoId: string | undefined = undefined; + runs.forEach((run: any) => { + if ('navigationEndpoint' in run && + 'watchEndpoint' in run.navigationEndpoint && + 'videoId' in run.navigationEndpoint.watchEndpoint) { + videoId = run.navigationEndpoint.watchEndpoint.videoId; + title = run.text; + } else if ('navigationEndpoint' in run && + 'browseEndpoint' in run.navigationEndpoint && + 'browseEndpointContextSupportedConfigs' in run.navigationEndpoint.browseEndpoint && + 'browseEndpointContextMusicConfig' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs && + 'pageType' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig && + run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig.pageType === 'MUSIC_PAGE_TYPE_ALBUM') { + album = { + url: `https://music.youtube.com/browse/${run.navigationEndpoint.browseEndpoint.browseId}`, + name: run.text, + } + } else if ('navigationEndpoint' in run && + 'browseEndpoint' in run.navigationEndpoint && + 'browseEndpointContextSupportedConfigs' in run.navigationEndpoint.browseEndpoint && + 'browseEndpointContextMusicConfig' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs && + 'pageType' in run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig && + run.navigationEndpoint.browseEndpoint.browseEndpointContextSupportedConfigs.browseEndpointContextMusicConfig.pageType === 'MUSIC_PAGE_TYPE_ARTIST') { + artist = { + url: `https://music.youtube.com/browse/${run.navigationEndpoint.browseEndpoint.browseId}`, + name: run.text, + } } }); + if(album.name && artist.name) { + album.artist = artist; + } return { title: title, url: `https://music.youtube.com/watch?v=${videoId}`, - artist: artists[0], - album: albums[0], + artist: artist, + album: album, } }) } catch (e) { - console.log("Error parsing songs:", e.message); + console.log("Error parsing tracks:", e.message); return []; } } -export function parseArtists(initialData: any): IntegrationTrack[] { +export function parseArtists(initialData: any): IntegrationArtist[] { try { var musicResponsiveListItemRenderers: any[] = []; diff --git a/server/integrations/integrations.ts b/server/integrations/integrations.ts index 1063f30..6e380be 100644 --- a/server/integrations/integrations.ts +++ b/server/integrations/integrations.ts @@ -45,6 +45,9 @@ export function createIntegrations(knex: Knex, apiBaseUrl: string) { let replaced = path.replace(new RegExp(`${apiBaseUrl}/integrations/[0-9]+/`), ''); console.log("Rewrite URL:", path, replaced); return replaced; + }, + onProxyReq: (proxyReq: any, req: any, res: any) => { + console.log('--> ', req.method, req.path, '->', proxyReq.path); } }); @@ -57,6 +60,9 @@ export function createIntegrations(knex: Knex, apiBaseUrl: string) { let replaced = path.replace(new RegExp(`${apiBaseUrl}/integrations/[0-9]+/`), ''); console.log("Rewrite URL:", path, replaced); return replaced; + }, + onProxyReq: (proxyReq: any, req: any, res: any) => { + console.log('--> ', req.method, req.path, '->', proxyReq.path); } })