ci: Deploy demo to appspot nightly and on releases (shaka-project#4038)

This imports our appspot code into the repo and updates all of it to
Python 3.

.github/workflows/custom-actions/prep-for-appspot/action.yaml

@@ -0,0 +1,56 @@
+name: Prep Appspot Deployment
+
+description: |
+  A reusable action that prepares the sources to deploy the Shaka Player Demo
+  to appspot.  Assumes that the source is already checked out.
+
+runs:
+  using: composite
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: npm ci
+
+    - name: Build Shaka Player
+      shell: bash
+      run: python build/all.py
+
+    - name: Extract git version
+      shell: bash
+      run: |
+        (
+          cd build
+          python3 -c 'from shakaBuildHelpers import git_version; print(git_version())'
+        ) > demo-version
+
+    - name: Tag the Player version
+      shell: bash
+      run: |
+        DEMO_VERSION="$(cat demo-version)-uncompiled"
+        sed \
+            -i lib/player.js \
+            -e "s/\(shaka.Player.version\) = .*/\1 = '$DEMO_VERSION';/"
+        rm demo-version
+
+    - name: Backup demo node modules
+      shell: bash
+      run: |
+        DEMO_NODE_MODULES=$(jq -r .shakaCustom.demoDeps[] package.json)
+        tar cf demo-modules.tar \
+            $(echo "$DEMO_NODE_MODULES" | sed -e 's@^@node_modules/@')
+
+    - name: Delete unneeded files
+      shell: bash
+      run: rm -rf .git .github build externs test node_modules
+
+    - name: Restore demo node modules
+      shell: bash
+      run: |
+        tar xf demo-modules.tar
+        rm demo-modules.tar
+
+    - name: Install App Engine code
+      shell: bash
+      run: |
+        mv app-engine/shaka-player-demo/* .
+        rm README.md

.github/workflows/demo-version-index.yaml

@@ -0,0 +1,29 @@
+name: Deploy Demo Version Index
+
+on:
+  workflow_dispatch:
+    # Allows for manual triggering.
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/demo-version-index.yaml
+      - app-engine/demo-version-index/**
+
+jobs:
+  appspot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.APPENGINE_DEPLOY_KEY }}'
+
+      - uses: google-github-actions/deploy-appengine@v0
+        with:
+          project_id: shaka-player-demo
+          version: index
+          working_directory: app-engine/demo-version-index/
+          promote: false
+

.github/workflows/nightly-demo.yaml

@@ -0,0 +1,42 @@
+name: Deploy Nightly Demo
+
+on:
+  workflow_dispatch:
+    # Allows for manual triggering.
+  schedule:
+    # Run every night at midnight PST / 8am UTC.
+    - cron: '0 8 * * *'
+
+jobs:
+  appspot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+
+      # The nightly demo has its own receiver app ID that points to the nightly
+      # demo itself for the receiver side.
+      - name: Override Cast Receiver App ID
+        run: |
+          sed \
+            -i demo/index.html \
+            -e 's/\(data-shaka-player-cast-receiver-id\)="[^"]*"/\1="07AEE832"/'
+
+      - uses: ./.github/workflows/custom-actions/prep-for-appspot
+
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.APPENGINE_DEPLOY_KEY }}'
+
+      - uses: google-github-actions/deploy-appengine@v0
+        with:
+          project_id: shaka-player-demo
+          version: nightly
+          promote: false

.github/workflows/release-please.yaml

@@ -93,4 +93,51 @@ jobs:
           file_glob: true
           overwrite: true
 
-  # TODO: Publish to appspot
+  appspot:
+    runs-on: ubuntu-latest
+    needs: release
+    if: ${{ needs.release.outputs.release_created }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+
+      - uses: ./.github/workflows/custom-actions/prep-for-appspot
+
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.APPENGINE_DEPLOY_KEY }}'
+
+      - name: Compute appspot subdomain
+        run: |
+          # This is the same as the version tag, but with dots replaced by
+          # dashes.  For example, v3.2.2 would have the subdomain v3-2-2.
+          APPSPOT_SUBDOMAIN=$( echo ${{ needs.release.outputs.tag_name }} | sed -e 's/\./-/' )
+          echo APPSPOT_SUBDOMAIN=$APPSPOT_SUBDOMAIN >> $GITHUB_ENV
+
+      - name: Compute appspot promotion
+        run: |
+          # "Promoting" an appspot deployment makes it the default which shows
+          # up on shaka-player-demo.appspot.com (no subdomain).  This should be
+          # done for the latest release version from the latest release branch.
+          RELEASE_TAGS=$(git tag | grep ^v[0-9] | grep -Ev -- '-(master|main)')
+          LATEST_RELEASE=$(echo "$RELEASE_TAGS" | sort --version-sort | tail -1)
+          TAG_NAME=${{ needs.release.outputs.tag_name }}
+
+          if [[ "$TAG_NAME" == "$LATEST_RELEASE" ]]; then
+            echo APPSPOT_PROMOTE=true >> $GITHUB_ENV
+          else
+            echo APPSPOT_PROMOTE=false >> $GITHUB_ENV
+          fi
+
+      - uses: google-github-actions/deploy-appengine@v0
+        with:
+          project_id: shaka-player-demo
+          version: ${{ env.APPSPOT_SUBDOMAIN }}
+          promote: ${{ env.APPSPOT_PROMOTE }}

app-engine/README.md

@@ -0,0 +1,9 @@
+# Google App Engine Code
+
+This folder contains source code for the services we run on Google App Engine
+(appspot.com).
+
+ - shaka-player-demo: Our hosted demo on https://shaka-player-demo.appspot.com/
+
+ - demo-version-index: An index of Shaka Player releases and demos, hosted at
+                       https://index-dot-shaka-player-demo.appspot.com/

app-engine/demo-version-index/README.md

@@ -0,0 +1,14 @@
+# Google App Engine Version Index
+
+This folder contains everything necessary to host an index of Shaka Player
+releases and demos at https://index-dot-shaka-player-demo.appspot.com/
+
+ - app.yaml: App Engine config file.  Defines the runtime (Python 3).
+
+ - main.py: A python service that queries available versions and generates the
+            index from a template.
+
+ - requirements.txt: Used by App Engine to install the necessary Python server
+                     requirements (Flask, App Engine API).
+
+ - templates/index.html: A Jinja2 template used to generate the index HTML.

app-engine/demo-version-index/app.yaml

@@ -0,0 +1,2 @@
+runtime: python39
+default_expiration: 1s

app-engine/demo-version-index/main.py

@@ -0,0 +1,162 @@
+# Shaka Player Version Index - Appspot Entrypoint
+# Copyright 2022 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
+# Generate an index of Shaka Player versions on appspot.
+
+import collections
+import google.appengine.api.modules
+import jinja2
+import os
+import random
+import re
+
+from flask import Flask, render_template
+
+DEMO_URL_TEMPLATE = 'https://{0}-dot-shaka-player-demo.appspot.com/'
+
+# In Google Hosted Libraries
+HOSTED_URL_TEMPLATE = 'https://ajax.googleapis.com/ajax/libs/shaka-player/{0}/shaka-player.{1}'
+
+# Before Google Hosted Libraries, v1 appspot URLs
+V1_URL_TEMPLATE = 'https://{0}-dot-shaka-player-demo.appspot.com/shaka-player.{1}'
+# Before Google Hosted Libraries, v2 appspot URLs
+V2_URL_TEMPLATE = 'https://{0}-dot-shaka-player-demo.appspot.com/dist/shaka-player.{1}'
+
+def version_to_demo_url(v):
+  return DEMO_URL_TEMPLATE.format(v.replace('.', '-'))
+
+def version_to_lib_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'compiled.js')
+  elif (version_key(v) == version_key('v1.6.5') or
+      version_key(v) >= version_key('v2.0.6')):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'compiled.js')
+  elif version_key(v) >= version_key('v2.0.0-beta'):
+    return V2_URL_TEMPLATE.format(v.replace('.', '-'), 'compiled.js')
+  else:
+    return V1_URL_TEMPLATE.format(v.replace('.', '-'), 'compiled.js')
+
+def version_to_ui_lib_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'ui.js')
+  elif version_key(v) >= version_key('v2.5.0'):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'ui.js')
+  else:
+    return None
+
+def version_to_lib_externs_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'compiled.externs.js')
+  elif version_key(v) >= version_key('v2.0.6'):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'compiled.externs.js')
+  else:
+    return None
+
+def version_to_ui_lib_externs_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'ui.externs.js')
+  elif version_key(v) >= version_key('v2.5.0'):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'ui.externs.js')
+  else:
+    return None
+
+def version_to_lib_defs_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'compiled.d.ts')
+  elif version_key(v) >= version_key('v3.0.6'):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'compiled.d.ts')
+  else:
+    return None
+
+def version_to_ui_lib_defs_url(v):
+  if v == 'nightly':
+    return V2_URL_TEMPLATE.format(v, 'ui.d.ts')
+  elif version_key(v) >= version_key('v3.0.6'):
+    return HOSTED_URL_TEMPLATE.format(v.replace('v', ''), 'ui.d.ts')
+  else:
+    return None
+
+def version_to_metadata(v):
+  return {
+    'version': v,
+    'best': False,  # Corrected later in another loop
+    'demo': version_to_demo_url(v),
+    'lib': version_to_lib_url(v),
+    'ui_lib': version_to_ui_lib_url(v),
+    'lib_externs': version_to_lib_externs_url(v),
+    'ui_lib_externs': version_to_ui_lib_externs_url(v),
+    'lib_defs': version_to_lib_defs_url(v),
+    'ui_lib_defs': version_to_ui_lib_defs_url(v),
+  }
+
+def is_release_version(name):
+  return re.match(r'v\d+-\d+-\d+(?:-.+)?', name)
+
+def appengine_version_to_package_version(version):
+  # Replace the first two dashes with dots.  More dashes indicate a prerelease
+  # version, as seen in "v2.0.0-beta3".
+  return version.replace('-', '.', 2)
+
+def version_key(version):
+  if version == 'nightly':
+    # A false version number for nightly, greater than any actual release
+    # version.
+    return [float('inf')]
+
+  assert version[0] == 'v'
+  main_version, _, suffix = version[1:].partition('-')
+  if not suffix:
+    suffix = '}}}'  # this puts main releases after prerelease versions
+  version_tuple = [int(x) for x in main_version.split('.')]
+  return version_tuple + [suffix]
+
+def get_appengine_versions():
+  if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'):
+    # NOTE: this doesn't return anything useful in a local dev server.
+    return google.appengine.api.modules.modules.get_versions()
+
+  # For a local dev server, fake it so we can test sorting.
+  fake_versions = [
+    'v1-6-0', 'v1-6-1', 'v1-6-2', 'v1-6-3', 'v1-6-4', 'v1-6-5',
+    'v2-0-0-beta', 'v2-0-0-beta2', 'v2-0-0-beta3', 'v2-0-0',
+    'v2-1-0', 'v2-1-1', 'v2-1-2',
+    'v2-2-1', 'v2-2-2-beta', 'v2-2-2-beta2', 'v2-2-2', 'v2-2-9', 'v2-2-10',
+  ]
+  random.shuffle(fake_versions)  # in-place shuffle
+  return fake_versions
+
+
+app = Flask(__name__)
+
+@app.route('/')
+def root():
+  appengine_versions = get_appengine_versions()
+  # Filter for release versions only.
+  appengine_versions = filter(is_release_version, appengine_versions)
+
+  # Now convert from appengine versions (v2-0-0) to package versions (v2.0.0).
+  versions = list(map(appengine_version_to_package_version, appengine_versions))
+  # Now sort, putting prerelease versions ahead of the corresponding release.
+  versions.sort(key=version_key)
+
+  version_metadata = collections.OrderedDict()
+  latest_by_branch = {}
+  for v in versions:
+    version_metadata[v] = version_to_metadata(v)
+
+    # Because |versions| is already sorted, we can just overwrite the entry
+    # in the latest_by_branch dictionary.
+    branch_key = tuple(version_key(v)[0:2])
+    latest_by_branch[branch_key] = v
+
+  for v in latest_by_branch.values():
+    version_metadata[v]['best'] = True
+
+  # Append nightly rather than filter for it, so that it always appears at the
+  # end of the list.
+  version_metadata['nightly'] = version_to_metadata('nightly')
+  version_metadata['nightly']['best'] = True
+
+  for i in version_metadata.values(): print(i)
+  return render_template('index.html', versions=version_metadata.values())

app-engine/demo-version-index/requirements.txt

@@ -0,0 +1,6 @@
+# Shaka Player Version Index - Appspot Python Runtime Requirements
+# Copyright 2022 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
+Flask==2.0.3
+appengine-python-standard==0.3.1