import React from 'react';
import Integration, { IntegrationFeature, IntegrationAlbum, IntegrationArtist, IntegrationSong } from '../Integration';
import StoreLinkIcon, { ExternalStore } from '../../../components/common/StoreLinkIcon';
enum SearchType {
Song = 'track',
Artist = 'artist',
Album = 'album',
};
export function extractInitialData(text: string): any | undefined {
// At the time of writing this, the scraper is trying to capture from the following block:
//
// initialData.push({
// path: ...,
// params: {"query":"something"},
// data: "THIS",
// });
//
// the THIS part.
// Get the whole line containing the data part.
let pattern = /initialData\.push\({[\n\r\s]*path:.*[\n\r\s]+params:\s*{\s*['"]query['"].*[\n\r\s]+data:\s*['"](.*)['"]\s*[\n\r]/
let m = text.match(pattern);
let dataline = Array.isArray(m) && m.length >= 2 ? m[1] : undefined;
if (!dataline) { return undefined; }
// Now parse the data line.
let dataline_clean = dataline.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"')
console.log(dataline);
console.log(dataline_clean);
let json = JSON.parse(dataline_clean);
return json;
}
export function parseSongs(initialData: any): IntegrationSong[] {
try {
var songMusicResponsiveListItemRenderers: any[] = [];
// Scrape for any "Song"-type items.
initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
if (c.musicShelfRenderer) {
c.musicShelfRenderer.contents.forEach((cc: any) => {
if (cc.musicResponsiveListItemRenderer &&
cc.musicResponsiveListItemRenderer.flexColumns &&
cc.musicResponsiveListItemRenderer.flexColumns[1]
.musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") {
songMusicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
}
})
}
})
return songMusicResponsiveListItemRenderers.map((s: any) => {
let videoId = s.doubleTapCommand.watchEndpoint.videoId;
let columns = s.flexColumns;
if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Song") {
throw new Error('song item doesnt match scraper expectation');
}
let title = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text;
let artists = columns[2].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
return 'navigationEndpoint' in run;
}).map((run: any) => {
let id = run.navigationEndpoint.browseEndpoint.browseId;
return {
url: `https://music.youtube.com/browse/${id}`,
name: run.text,
}
});
let albums = columns[3].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
return 'navigationEndpoint' in run;
}).map((run: any) => {
let id = run.navigationEndpoint.browseEndpoint.browseId;
return {
url: `https://music.youtube.com/browse/${id}`,
name: run.text,
artist: artists[0],
}
});
return {
title: title,
url: `https://music.youtube.com/watch?v=${videoId}`,
artist: artists[0],
album: albums[0],
}
})
} catch (e) {
console.log("Error parsing songs:", e.message);
return [];
}
}
export default class YoutubeMusicWebScraper extends Integration {
integrationId: number;
constructor(integrationId: number) {
super(integrationId);
this.integrationId = integrationId;
}
getFeatures(): IntegrationFeature[] {
return [
IntegrationFeature.Test,
IntegrationFeature.SearchSong,
IntegrationFeature.SearchAlbum,
IntegrationFeature.SearchArtist,
]
}
getIcon(props: any) {
return
}
providesStoreLink() {
return ExternalStore.YoutubeMusic;
}
async test(testParams: {}) {
const response = await fetch(
(process.env.REACT_APP_BACKEND || "") +
`/integrations/${this.integrationId}/search?q=${encodeURIComponent('No One Knows Queens Of The Stone Age')}`);
let text = await response.text();
let songs = parseSongs(extractInitialData(text));
console.log("Found songs", songs);
if (!Array.isArray(songs) || songs.length === 0 || songs[0].title !== "No One Knows") {
throw new Error("Test failed; No One Knows was not correctly identified.");
}
}
async searchSong(query: string, limit: number): Promise {
const response = await fetch(
(process.env.REACT_APP_BACKEND || "") +
`/integrations/${this.integrationId}/search?q=${encodeURIComponent(query)}`);
let text = await response.text();
return parseSongs(extractInitialData(text));
}
async searchAlbum(query: string, limit: number): Promise { return []; }
async searchArtist(query: string, limit: number): Promise { return []; }
async search(query: string, type: SearchType, limit: number):
Promise {
return [];
// const response = await fetch(
// (process.env.REACT_APP_BACKEND || "") +
// `/integrations/${this.integrationId}/v1/search?q=${encodeURIComponent(query)}&type=${type}&limit=${limit}`);
// if (!response.ok) {
// throw new Error("Spotify Client Credentials search failed: " + JSON.stringify(response));
// }
// let json = await response.json();
// console.log("Response:", json);
// switch(type) {
// case SearchType.Song: {
// return json.tracks.items.map((r: any): IntegrationSong => {
// return {
// title: r.name,
// url: r.external_urls.spotify,
// artist: {
// name: r.artists && r.artists[0].name,
// url: r.artists && r.artists[0].external_urls.spotify,
// },
// album: {
// name: r.album && r.album.name,
// url: r.album && r.album.external_urls.spotify,
// }
// }
// })
// }
// case SearchType.Artist: {
// return json.artists.items.map((r: any): IntegrationArtist => {
// return {
// name: r.name,
// url: r.external_urls.spotify,
// }
// })
// }
// case SearchType.Album: {
// return json.albums.items.map((r: any): IntegrationAlbum => {
// return {
// name: r.name,
// url: r.external_urls.spotify,
// artist: {
// name: r.artists[0].name,
// url: r.artists[0].external_urls.spotify,
// },
// }
// })
// }
}
}