import React from 'react'; import Integration, { IntegrationFeature, IntegrationAlbum, IntegrationArtist, IntegrationSong } from '../Integration'; import StoreLinkIcon, { ExternalStore } from '../../../components/common/StoreLinkIcon'; enum SearchType { Song = 'track', Artist = 'artist', Album = 'album', }; export function extractInitialData(text: string): any | undefined { // At the time of writing this, the scraper is trying to capture from the following block: // // initialData.push({ // path: ..., // params: {"query":"something"}, // data: "THIS", // }); // // the THIS part. // Get the whole line containing the data part. let pattern = /initialData\.push\({[\n\r\s]*path:.*[\n\r\s]+params:\s*{\s*['"]query['"].*[\n\r\s]+data:\s*['"](.*)['"]\s*[\n\r]/ let m = text.match(pattern); let dataline = Array.isArray(m) && m.length >= 2 ? m[1] : undefined; if (!dataline) { return undefined; } // Now parse the data line. let dataline_clean = dataline.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"') console.log(dataline); console.log(dataline_clean); let json = JSON.parse(dataline_clean); return json; } export function parseSongs(initialData: any): IntegrationSong[] { try { var songMusicResponsiveListItemRenderers: any[] = []; // Scrape for any "Song"-type items. initialData.contents.sectionListRenderer.contents.forEach((c: any) => { if (c.musicShelfRenderer) { c.musicShelfRenderer.contents.forEach((cc: any) => { if (cc.musicResponsiveListItemRenderer && cc.musicResponsiveListItemRenderer.flexColumns && cc.musicResponsiveListItemRenderer.flexColumns[1] .musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") { songMusicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer); } }) } }) return songMusicResponsiveListItemRenderers.map((s: any) => { let videoId = s.doubleTapCommand.watchEndpoint.videoId; let columns = s.flexColumns; if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Song") { throw new Error('song item doesnt match scraper expectation'); } let title = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text; let artists = columns[2].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => { return 'navigationEndpoint' in run; }).map((run: any) => { let id = run.navigationEndpoint.browseEndpoint.browseId; return { url: `https://music.youtube.com/browse/${id}`, name: run.text, } }); let albums = columns[3].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => { return 'navigationEndpoint' in run; }).map((run: any) => { let id = run.navigationEndpoint.browseEndpoint.browseId; return { url: `https://music.youtube.com/browse/${id}`, name: run.text, artist: artists[0], } }); return { title: title, url: `https://music.youtube.com/watch?v=${videoId}`, artist: artists[0], album: albums[0], } }) } catch (e) { console.log("Error parsing songs:", e.message); return []; } } export default class YoutubeMusicWebScraper extends Integration { integrationId: number; constructor(integrationId: number) { super(integrationId); this.integrationId = integrationId; } getFeatures(): IntegrationFeature[] { return [ IntegrationFeature.Test, IntegrationFeature.SearchSong, IntegrationFeature.SearchAlbum, IntegrationFeature.SearchArtist, ] } getIcon(props: any) { return } providesStoreLink() { return ExternalStore.YoutubeMusic; } async test(testParams: {}) { const response = await fetch( (process.env.REACT_APP_BACKEND || "") + `/integrations/${this.integrationId}/search?q=${encodeURIComponent('No One Knows Queens Of The Stone Age')}`); let text = await response.text(); let songs = parseSongs(extractInitialData(text)); console.log("Found songs", songs); if (!Array.isArray(songs) || songs.length === 0 || songs[0].title !== "No One Knows") { throw new Error("Test failed; No One Knows was not correctly identified."); } } async searchSong(query: string, limit: number): Promise { const response = await fetch( (process.env.REACT_APP_BACKEND || "") + `/integrations/${this.integrationId}/search?q=${encodeURIComponent(query)}`); let text = await response.text(); return parseSongs(extractInitialData(text)); } async searchAlbum(query: string, limit: number): Promise { return []; } async searchArtist(query: string, limit: number): Promise { return []; } async search(query: string, type: SearchType, limit: number): Promise { return []; // const response = await fetch( // (process.env.REACT_APP_BACKEND || "") + // `/integrations/${this.integrationId}/v1/search?q=${encodeURIComponent(query)}&type=${type}&limit=${limit}`); // if (!response.ok) { // throw new Error("Spotify Client Credentials search failed: " + JSON.stringify(response)); // } // let json = await response.json(); // console.log("Response:", json); // switch(type) { // case SearchType.Song: { // return json.tracks.items.map((r: any): IntegrationSong => { // return { // title: r.name, // url: r.external_urls.spotify, // artist: { // name: r.artists && r.artists[0].name, // url: r.artists && r.artists[0].external_urls.spotify, // }, // album: { // name: r.album && r.album.name, // url: r.album && r.album.external_urls.spotify, // } // } // }) // } // case SearchType.Artist: { // return json.artists.items.map((r: any): IntegrationArtist => { // return { // name: r.name, // url: r.external_urls.spotify, // } // }) // } // case SearchType.Album: { // return json.albums.items.map((r: any): IntegrationAlbum => { // return { // name: r.name, // url: r.external_urls.spotify, // artist: { // name: r.artists[0].name, // url: r.artists[0].external_urls.spotify, // }, // } // }) // } } }