From 74cee65ab44ae815721b0c3dcf1ba38ee7cc1569 Mon Sep 17 00:00:00 2001 From: nerix Date: Wed, 30 Jun 2021 14:10:39 +0200 Subject: [PATCH] feat(ads): HLS-Merging (#8) * docs: :memo: Mark todo item as done * feat(ads): :sparkles: initial hls-merging * fix: switch segments --- README.md | 2 +- src/background/ad.replacement.ts | 24 ++--- src/background/background.ts | 11 ++- src/background/gql.requests.ts | 42 ++++++++- src/background/hls/hls-merger.ts | 26 ++++++ src/background/hls/hls-parser.ts | 108 +++++++++++++++++++++++ src/background/hls/hls-segments-tags.ts | 112 ++++++++++++++++++++++++ src/background/hls/hls-types.ts | 16 ++++ src/background/hls/hls-writer.ts | 57 ++++++++++++ src/background/player-merging.ts | 47 ++++++++++ src/background/storage.ts | 18 ++++ 11 files changed, 437 insertions(+), 26 deletions(-) create mode 100644 src/background/hls/hls-merger.ts create mode 100644 src/background/hls/hls-parser.ts create mode 100644 src/background/hls/hls-segments-tags.ts create mode 100644 src/background/hls/hls-types.ts create mode 100644 src/background/hls/hls-writer.ts create mode 100644 src/background/player-merging.ts create mode 100644 src/background/storage.ts diff --git a/README.md b/README.md index d790c6a..8e186b9 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Downloads can be found on the [releases-page](https://github.com/Nerixyz/ttv-too * Settings-Panel on the player * Add tests for ad-skipping/-matching * Provide better feedback -* Possibly parse playlist +* ~~Possibly parse playlist~~ * Investigate [`swc`/`spack`](https://swc.rs/) for faster compilation (blocking: [#1438](https://github.com/swc-project/swc/issues/1438)) # Set-Up diff --git a/src/background/ad.replacement.ts b/src/background/ad.replacement.ts index cb9a446..6ce726e 100644 --- a/src/background/ad.replacement.ts +++ b/src/background/ad.replacement.ts @@ -1,6 +1,7 @@ import { TwitchStitchedAdData } from './twitch-m3u8.types'; -import { getPlayerAccessTokenRequest, makeAdRequest } from './gql.requests'; +import { getPlayerAccessTokenRequest, makeAdRequest, PlayerType } from './gql.requests'; import { eventHandler } from './utilities/messaging'; +import { getUsherData } from './storage'; export async function onAdPod(stitchedAd: TwitchStitchedAdData, stream: string) { const adPod = { @@ -16,26 +17,11 @@ export async function onAdPod(stitchedAd: TwitchStitchedAdData, stream: string) }; await makeAdRequest(adPod); - let {usherData} = await browser.storage.local.get('usherData'); - if(!usherData) { - console.warn('No usherData pogo, replacing'); - usherData = { - allow_source: "true", - fast_bread: "true", - player_backend: "mediaplayer", - playlist_include_framerate: "true", - reassignments_supported: "true", - supported_codecs: "avc1", - cdm: "wv", - player_version: "1.2.0" - } - } - - eventHandler.emitContext('updateUrl',{url: await createM3U8Url({stream, usher: usherData}), stream}); + eventHandler.emitContext('updateUrl',{url: await createM3U8Url({stream, usher: await getUsherData()}), stream}); } -async function createM3U8Url({usher, stream}: {usher: any, stream: string}) { - const {value: token, signature: sig} = await getPlayerAccessTokenRequest(stream); +export async function createM3U8Url({usher, stream}: {usher: any, stream: string}, playerType = PlayerType.Site) { + const {value: token, signature: sig} = await getPlayerAccessTokenRequest(stream, playerType); const search = new URLSearchParams({ ...usher, diff --git a/src/background/background.ts b/src/background/background.ts index e960f38..b079738 100644 --- a/src/background/background.ts +++ b/src/background/background.ts @@ -6,6 +6,9 @@ import { onAdPod, StreamTabs } from './ad.replacement'; import { TWITCH_USER_PAGE } from './utilities/request.utilities'; import { eventHandler } from './utilities/messaging'; import { TwitchStitchedAdData } from './twitch-m3u8.types'; +import { parseMediaPlaylist } from './hls/hls-parser'; +import { writePlaylist } from './hls/hls-writer'; +import { mergePlayer } from './player-merging'; function onRequest(request: _OnBeforeRequestDetails) { if (!request.url.includes('video-weaver')) return; @@ -17,9 +20,9 @@ function onRequest(request: _OnBeforeRequestDetails) { filter.ondata = async (event: { data: ArrayBuffer }) => { const text = decoder.decode(event.data); - const adIdx = text.indexOf('#EXT-X-DATERANGE:ID="stitched-ad'); + const hasAds = text.match(/^#EXTINF:\d+\.?\d*,[^live]+$/m); - if (adIdx === -1) { + if (!hasAds) { filter.write(event.data); return; } @@ -28,9 +31,9 @@ function onRequest(request: _OnBeforeRequestDetails) { filter.write(encoder.encode(data)); filter.close(); }; - extractAdData(text, request.documentUrl ?? '', request.tabId); - finalWrite(cleanupAllAdStuff(text)); + extractAdData(text, request.documentUrl ?? '', request.tabId); + finalWrite(await mergePlayer(request.tabId, text)); }; filter.onstop = () => { diff --git a/src/background/gql.requests.ts b/src/background/gql.requests.ts index dba0124..8a5cbdd 100644 --- a/src/background/gql.requests.ts +++ b/src/background/gql.requests.ts @@ -99,12 +99,12 @@ query PlaybackAccessToken_Template($login: String!, $isLive: Boolean!, $vodID: I } `.replace(/[\n\r]/g, ''); -export async function getPlayerAccessTokenRequest(login: string): Promise<{value: string, signature: string}> { +export async function getPlayerAccessTokenRequest(login: string, playerType = PlayerType.Site): Promise<{value: string, signature: string}> { const res = await gqlRequest(makeRawGqlPacket('PlaybackAccessToken', PLAYBACK_ACCESS_TOKEN_QUERY, { isLive: true, isVod: false, login, - playerType: 'site', + playerType, vodID: '' })).then(x => x.json()); const accessToken = res?.data?.streamPlaybackAccessToken; @@ -113,3 +113,41 @@ export async function getPlayerAccessTokenRequest(login: string): Promise<{value delete accessToken.__typename; return accessToken; } + +export enum PlayerType { + AmazonLive = 'amazon_live', + AmazonProductPage = 'amazon_product_page', + AmazonVse = 'amazon_vse_test', + AnimatedThumbnails = 'animated_thumbnails', + ChannelTrailer = 'channel_trailer', + ClipsEditing = 'clips-editing', + ClipsEmbed = 'clips-embed', + ClipsViewing = 'clips-viewing', + ClipsWatchPage = 'clips-watch', + Creative = 'creative', + Curse = 'curse', + Dashboard = 'dashboard', + VideoProducerModal = 'video_producer_modal', + Embed = 'embed', + Facebook = 'facebook', + Feed = 'feed', + Frontpage = 'frontpage', + Highlighter = 'highlighter', + Imdb = 'imdb', + MultiviewPrimary = 'multiview-primary', + MultiviewSecondary = 'multiview-secondary', + Onboarding = 'onboarding', + PictureByPicture = 'picture-by-picture', + Popout = 'popout', + Pulse = 'pulse', + Site = 'site', + Thunderdome = 'thunderdome', + ChannelHomeCarousel = 'channel_home_carousel', + ChannelHomeLive = 'channel_home_live', + SiteMini = 'site_mini', + SquadPrimary = 'squad_primary', + SquadSecondary = 'squad_secondary', + TwitchEverywhere = 'twitch_everywhere', + WatchPartyHost = 'watch_party_host', + PopTart = 'pop_tart' +} diff --git a/src/background/hls/hls-merger.ts b/src/background/hls/hls-merger.ts new file mode 100644 index 0000000..bd877a2 --- /dev/null +++ b/src/background/hls/hls-merger.ts @@ -0,0 +1,26 @@ +import { HlsPlaylist, MediaSegment } from "./hls-types"; + +/** + * Note: this replaces segments in the target playlist + * @param target The playlist to be merged into + * @param reference The playlist which provides live segments + * @returns The "new" playlist + */ +export function mergePlaylists(target: HlsPlaylist, reference: HlsPlaylist) { + const findRef = (segment: MediaSegment) => { + if (!segment.dateTime) return; + return reference.segments.find(ref => ref.dateTime && ref.dateTime!.date >= segment.dateTime!.date); + } + + for (const segment of target.segments) { + if (segment.segmentInfo.title !== 'live') { + const replacement = findRef(segment); + if (replacement) { + segment.uri = replacement.uri; + segment.dateTime = replacement.dateTime; + } + } + } + + return target; +} \ No newline at end of file diff --git a/src/background/hls/hls-parser.ts b/src/background/hls/hls-parser.ts new file mode 100644 index 0000000..a241b74 --- /dev/null +++ b/src/background/hls/hls-parser.ts @@ -0,0 +1,108 @@ +import { ExtInfTag, ExtXDaterange, ExtXDiscontinuity, ExtXProgramDateTime, SegmentTag } from "./hls-segments-tags"; +import { HlsPlaylist } from "./hls-types"; + +/** + * Note: This isn't a full hls parser. It only parses media playlists. + * + * @param raw The playlist file + */ +export function parseMediaPlaylist(raw: string): HlsPlaylist { + const lines = raw + .split('\n') + // ignore empty lines and comments + .filter(line => line && line.match(/^#EXT|[^#]/)); + + const isExtM3u = lines.shift() === '#EXTM3U'; + if (!isExtM3u) throw new Error('Invalid playlist'); + + const playlist: HlsPlaylist = { + version: 1, + dateranges: [], + tags: [], + segments: [], + twitchPrefech: [], + }; + + let segmentAttributes: Array> | null = null; + let segmentInfo: ExtInfTag | null = null; + let segmentDateTime: ExtXProgramDateTime | null = null; + + const pushSegmentAttr = (attr: SegmentTag) => (segmentAttributes ??= []).push(attr); + const flushAttr = () => { + const attr = segmentAttributes; + segmentAttributes = null; + return attr; + }; + + for (const next of lines) { + if (!next) continue; + + if (next?.startsWith('#')) { + const match = next.match(/^#(?[\w-]+):?(?.*)$/)?.groups; + if (!match) { + console.warn(`Invalid tag: ${next}`); + continue; + } + const { tag, attributes } = match; + + switch (tag) { + case 'EXT-X-VERSION': { + playlist.version = Number(attributes); + continue; + } + case 'EXTINF': { + segmentInfo = new ExtInfTag(attributes); + continue; + } + case 'EXT-X-PROGRAM-DATE-TIME': { + segmentDateTime = new ExtXProgramDateTime(attributes); + continue; + } + case 'EXT-X-DATERANGE': { + playlist.dateranges.push(new ExtXDaterange(attributes)); + continue; + } + case 'EXT-X-DISCONTINUITY': { + pushSegmentAttr(new ExtXDiscontinuity(attributes)); + continue; + } + case 'EXT-X-TWITCH-PREFETCH': { + playlist.twitchPrefech.push(next); + continue; + } + default: { + playlist.tags.push(next); + continue; + } + } + } else { + // a URI + + if (!segmentInfo) throw new Error('invalid playlist'); + + const attr = flushAttr(); + playlist.segments.push({ uri: next, attributes: attr ?? [], segmentInfo, dateTime: segmentDateTime }); + } + } + + return playlist; +} + +/** + * + * @param raw The attribute string - essentially: `'CLASS="twitch-stitched-ad",DURATION=15.187,...'` + * @returns The object-version: `{ CLASS: 'twitch-stitched-ad', DURATION: 15.187 }` + */ +export function parseAttributeList = Record>(raw: string): T { + return Object.fromEntries( + raw + .split(/(?:^|,)([^=]*=(?:"[^"]*"|[^,]*))/) + .filter(Boolean) + .map(x => { + const idx = x.indexOf('='); + const key = x.substring(0, idx); + const value = x.substring(idx + 1); + const num = Number(value); + return [key, Number.isNaN(num) ? value.startsWith('"') ? JSON.parse(value) : value : num] + })) as T; +} \ No newline at end of file diff --git a/src/background/hls/hls-segments-tags.ts b/src/background/hls/hls-segments-tags.ts new file mode 100644 index 0000000..b60916d --- /dev/null +++ b/src/background/hls/hls-segments-tags.ts @@ -0,0 +1,112 @@ +import { parseAttributeList } from "./hls-parser"; +import { writeAttributes } from "./hls-writer"; + +export abstract class SegmentTag { + abstract readonly TAGNAME: string; + + protected readonly attributes: Attr; + + constructor(rawAttributes: string) { + this.attributes = this.parseAttributes(rawAttributes); + } + + parseAttributes(raw: string): Attr { + // the implementation has to override it + return undefined as any as Attr; + } + + get attributeString(): string | null { + return null; + } + + [Symbol.toStringTag]() { + const attr = this.attributeString; + return `#${this.TAGNAME}${attr ? ':' + attr : ''}`; + } +} + +export class ExtInfTag extends SegmentTag<[number, string?]> { + TAGNAME = 'EXTINF'; + + get duration() { + return this.attributes[0]; + } + + get title() { + return this.attributes[1]; + } + + get attributeString() { + return `${this.duration.toFixed(3)}${this.title ? ',' + this.title : ''}`; + } + + parseAttributes(raw: string): [number, string?] { + const firstIdx = raw.indexOf(','); + const duration = Number(raw.substring(0, firstIdx)); + const title = raw.substring(firstIdx + 1); + + return [duration, title || undefined]; + } +} + +export class ExtXDiscontinuity extends SegmentTag { + TAGNAME = 'EXT-X-DISCONTINUITY'; +} + +export class ExtXProgramDateTime extends SegmentTag { + TAGNAME = 'EXT-X-PROGRAM-DATE-TIME'; + + get attributeString() { + return this.attributes.toISOString(); + } + + get date() { + return this.attributes; + } + + parseAttributes(raw: string) { + return new Date(raw); + } +} +export type ExtXDaterangeAttributes = { + ID: string; + CLASS?: string; + 'START-DATE': string; + 'END-DATE'?: string; + // DURATION?: number; -- not on twitch + // 'PLANNED-DURATION'?: number; + 'END-ON-NEXT'?: 'YES' + + [x: string]: any +} +export class ExtXDaterange extends SegmentTag { + TAGNAME = 'EXT-X-DATERANGE'; + + get id() { + return this.attributes.ID; + } + + get class() { + return this.attributes.CLASS; + } + + get start() { + return new Date(this.attributes["START-DATE"]); + } + + get end() { + return this.attributes["END-DATE"] ? new Date(this.attributes["END-DATE"]) : undefined; + } + + get endOnNext() { + return !!this.attributes["END-ON-NEXT"]; + } + + get attributeString() { + return writeAttributes(this.attributes); + } + + parseAttributes(raw: string) { + return parseAttributeList(raw); + } +} \ No newline at end of file diff --git a/src/background/hls/hls-types.ts b/src/background/hls/hls-types.ts new file mode 100644 index 0000000..314d4cd --- /dev/null +++ b/src/background/hls/hls-types.ts @@ -0,0 +1,16 @@ +import { ExtInfTag, ExtXDaterange, ExtXProgramDateTime, SegmentTag } from "./hls-segments-tags"; + +export interface HlsPlaylist { + version: number; + dateranges: ExtXDaterange[]; + tags: string[]; + segments: MediaSegment[]; + twitchPrefech: string[]; +} + +export interface MediaSegment { + uri: string; + attributes: SegmentTag[] | null; + segmentInfo: ExtInfTag; + dateTime: ExtXProgramDateTime | null; +} diff --git a/src/background/hls/hls-writer.ts b/src/background/hls/hls-writer.ts new file mode 100644 index 0000000..a902f61 --- /dev/null +++ b/src/background/hls/hls-writer.ts @@ -0,0 +1,57 @@ +import { SegmentTag } from "./hls-segments-tags"; +import { HlsPlaylist, MediaSegment } from "./hls-types"; + +export function writePlaylist(playlist: HlsPlaylist): string { + const lines: string[] = []; + + const pushTag = (tag: SegmentTag | null | undefined) => { + if (!tag) return; + const attributes = tag.attributeString; + lines.push(`#${tag.TAGNAME}${attributes ? ':' + attributes : ''}`); + } + + lines.push('#EXTM3U'); + lines.push(`#EXT-X-VERSION:${playlist.version}`); + lines.push(...playlist.tags); + + let nextDaterange = 0; + const peekDaterange = () => playlist.dateranges[nextDaterange]; + const consumeDaterange = () => playlist.dateranges[nextDaterange++]; + const checkAddDateranges = (segment: MediaSegment) => { + while (peekDaterange()?.start <= (segment.dateTime?.date ?? new Date() /* then always define before */)) { + pushTag(consumeDaterange()); + } + }; + + for (const segment of playlist.segments) { + checkAddDateranges(segment); + + if (segment.attributes) { + for (const attribute of segment.attributes) { + pushTag(attribute); + } + } + + pushTag(segment.dateTime); + pushTag(segment.segmentInfo); + lines.push(segment.uri); + } + + lines.push(...playlist.twitchPrefech); + + return lines.join('\n'); +} + +export function writeAttributes(attr: Record) { + return Object + .entries(attr) + .map(([k, v]) => + [k, + typeof v === "number" + ? v.toString() + : v === 'YES' + ? v + : `"${v}"` + ].join('=')) + .join(','); +} \ No newline at end of file diff --git a/src/background/player-merging.ts b/src/background/player-merging.ts new file mode 100644 index 0000000..8ca9dcd --- /dev/null +++ b/src/background/player-merging.ts @@ -0,0 +1,47 @@ +import { createM3U8Url, StreamTabs } from "./ad.replacement"; +import { PlayerType } from "./gql.requests"; +import { mergePlaylists } from "./hls/hls-merger"; +import { parseMediaPlaylist } from "./hls/hls-parser"; +import { writePlaylist } from "./hls/hls-writer"; +import { getUsherData } from "./storage"; + +const playlistUrls = new Map(); + +export async function mergePlayer(tabId: number, raw: string): Promise { + const stream = StreamTabs.get(tabId); + if (!stream) { + console.warn(`No stream in tab ${tabId}`); + return raw; + } + + if (!playlistUrls.has(tabId)) { + // get playlist + const m3u8Url = await createM3U8Url({ stream, usher: await getUsherData() }, PlayerType.PictureByPicture); + + const streamM3u8 = await fetch(m3u8Url).then(x => x.text()); + const [url] = streamM3u8.match(/^https:\/\/.+\.m3u8$/m) ?? []; + + if (!url) { + console.warn('could not get stream url'); + return raw; + } + + playlistUrls.set(tabId, url); + } + + const refRaw = await fetch(playlistUrls.get(tabId)!).then(x => x.text()); + + try { + const rawP = parseMediaPlaylist(raw); + // const refRawP = parseMediaPlaylist(refRaw); + // const merged = mergePlaylists(rawP, refRawP); + if (rawP.segments.find(x => x.dateTime && x.dateTime.date > new Date())?.segmentInfo?.title !== 'live') { + return refRaw; + } else { + return raw; + } + } catch (e) { + console.error(e); + return raw; + } +} diff --git a/src/background/storage.ts b/src/background/storage.ts new file mode 100644 index 0000000..af6b847 --- /dev/null +++ b/src/background/storage.ts @@ -0,0 +1,18 @@ +export async function getUsherData() { + let {usherData} = await browser.storage.local.get('usherData'); + if(!usherData) { + console.warn('No usherData pogo, replacing'); + usherData = { + allow_source: "true", + fast_bread: "true", + player_backend: "mediaplayer", + playlist_include_framerate: "true", + reassignments_supported: "true", + supported_codecs: "avc1", + cdm: "wv", + player_version: "1.3.0" + } + } + + return usherData; +} \ No newline at end of file