MuDBase/client/src/lib/integration/youtubemusic/YoutubeMusicWebScraper.tsx

import React from 'react';
import Integration, { IntegrationFeature, IntegrationAlbum, IntegrationArtist, IntegrationSong } from '../Integration';
import StoreLinkIcon, { ExternalStore } from '../../../components/common/StoreLinkIcon';

enum SearchType {
    Song = 'track',
    Artist = 'artist',
    Album = 'album',
};

export function extractInitialData(text: string): any | undefined {
    // At the time of writing this, the scraper is trying to capture from the following block:
    //
    // initialData.push({
    //    path: ...,
    //    params: {"query":"something"},
    //    data: "THIS",
    //  });
    //
    // the THIS part.

    // Get the whole line containing the data part.
    let pattern = /initialData\.push\({[\n\r\s]*path:.*[\n\r\s]+params:\s*{\s*['"]query['"].*[\n\r\s]+data:\s*['"](.*)['"]\s*[\n\r]/
    let m = text.match(pattern);
    let dataline = Array.isArray(m) && m.length >= 2 ? m[1] : undefined;
    if (!dataline) { return undefined; }

    // Now parse the data line.
    let dataline_clean = dataline.replace(/\\"/g, '"').replace(/\\\\"/g, '\\"')
    console.log(dataline);
    console.log(dataline_clean);

    let json = JSON.parse(dataline_clean);
    return json;
}

export function parseSongs(initialData: any): IntegrationSong[] {
    try {
        var songMusicResponsiveListItemRenderers: any[] = [];

        // Scrape for any "Song"-type items.
        initialData.contents.sectionListRenderer.contents.forEach((c: any) => {
            if (c.musicShelfRenderer) {
                c.musicShelfRenderer.contents.forEach((cc: any) => {
                    if (cc.musicResponsiveListItemRenderer &&
                        cc.musicResponsiveListItemRenderer.flexColumns &&
                        cc.musicResponsiveListItemRenderer.flexColumns[1]
                            .musicResponsiveListItemFlexColumnRenderer.text.runs[0].text === "Song") {
                                songMusicResponsiveListItemRenderers.push(cc.musicResponsiveListItemRenderer);
                            }
                })
            }
        })

        return songMusicResponsiveListItemRenderers.map((s: any) => {
            let videoId = s.doubleTapCommand.watchEndpoint.videoId;
            let columns = s.flexColumns;

            if (columns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text !== "Song") {
                throw new Error('song item doesnt match scraper expectation');
            }
            let title = columns[0].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text;

            let artists = columns[2].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
                return 'navigationEndpoint' in run;
            }).map((run: any) => {
                let id = run.navigationEndpoint.browseEndpoint.browseId;
                return {
                    url: `https://music.youtube.com/browse/${id}`,
                    name: run.text,
                }
            });

            let albums = columns[3].musicResponsiveListItemFlexColumnRenderer.text.runs.filter((run: any) => {
                return 'navigationEndpoint' in run;
            }).map((run: any) => {
                let id = run.navigationEndpoint.browseEndpoint.browseId;
                return {
                    url: `https://music.youtube.com/browse/${id}`,
                    name: run.text,
                    artist: artists[0],
                }
            });

            return {
                title: title,
                url: `https://music.youtube.com/watch?v=${videoId}`,
                artist: artists[0],
                album: albums[0],
            }
        })
    } catch (e) {
        console.log("Error parsing songs:", e.message);
        return [];
    }
}

export default class YoutubeMusicWebScraper extends Integration {
    integrationId: number;

    constructor(integrationId: number) {
        super(integrationId);
        this.integrationId = integrationId;
    }

    getFeatures(): IntegrationFeature[] {
        return [
            IntegrationFeature.Test,
            IntegrationFeature.SearchSong,
            IntegrationFeature.SearchAlbum,
            IntegrationFeature.SearchArtist,
        ]
    }

    getIcon(props: any) {
        return <StoreLinkIcon whichStore={ExternalStore.YoutubeMusic} {...props} />
    }

    providesStoreLink() {
        return ExternalStore.YoutubeMusic;
    }

    async test(testParams: {}) {
        const response = await fetch(
            (process.env.REACT_APP_BACKEND || "") +
            `/integrations/${this.integrationId}/search?q=${encodeURIComponent('No One Knows Queens Of The Stone Age')}`);

        let text = await response.text();
        let songs = parseSongs(extractInitialData(text));

        console.log("Found songs", songs);

        if (!Array.isArray(songs) || songs.length === 0 || songs[0].title !== "No One Knows") {
            throw new Error("Test failed; No One Knows was not correctly identified.");
        }
    }

    async searchSong(query: string, limit: number): Promise<IntegrationSong[]> {
        const response = await fetch(
            (process.env.REACT_APP_BACKEND || "") +
            `/integrations/${this.integrationId}/search?q=${encodeURIComponent(query)}`);

        let text = await response.text();
        return parseSongs(extractInitialData(text));
    }
    async searchAlbum(query: string, limit: number): Promise<IntegrationAlbum[]> { return []; }
    async searchArtist(query: string, limit: number): Promise<IntegrationArtist[]> { return []; }

    async search(query: string, type: SearchType, limit: number):
        Promise<IntegrationSong[] | IntegrationAlbum[] | IntegrationArtist[]> {

        return [];
        // const response = await fetch(
        //     (process.env.REACT_APP_BACKEND || "") +
        //     `/integrations/${this.integrationId}/v1/search?q=${encodeURIComponent(query)}&type=${type}&limit=${limit}`);

        // if (!response.ok) {
        //     throw new Error("Spotify Client Credentials search failed: " + JSON.stringify(response));
        // }

        // let json = await response.json();

        // console.log("Response:", json);

        // switch(type) {
        //     case SearchType.Song: {
        //         return json.tracks.items.map((r: any): IntegrationSong => {
        //             return {
        //                 title: r.name,
        //                 url: r.external_urls.spotify,
        //                 artist: {
        //                     name: r.artists && r.artists[0].name,
        //                     url: r.artists && r.artists[0].external_urls.spotify,
        //                 },
        //                 album: {
        //                     name: r.album && r.album.name,
        //                     url: r.album && r.album.external_urls.spotify,
        //                 }
        //             }
        //         })
        //     }
        //     case SearchType.Artist: {
        //         return json.artists.items.map((r: any): IntegrationArtist => {
        //             return {
        //                 name: r.name,
        //                 url: r.external_urls.spotify,
        //             }
        //         })
        //     }
        //     case SearchType.Album: {
        //         return json.albums.items.map((r: any): IntegrationAlbum => {
        //             return {
        //                 name: r.name,
        //                 url: r.external_urls.spotify,
        //                 artist: {
        //                     name: r.artists[0].name,
        //                     url: r.artists[0].external_urls.spotify,
        //                 },
        //             }
        //         })
        //     }
    }
}