feat: implement discover route (#36)

* feat: implement `discover` route

* chore(test): tidy up

* chore: remove empty line

* fix: strip code no longer required for helix 5

Author: dominique-pfister

src/cache/handler.js

@@ -13,7 +13,7 @@ import { Response } from '@adobe/fetch';
 import purge, { PURGE_PREVIEW_AND_LIVE } from './purge.js';
 
 /**
- * Allowed methods for that handler.
+ * Allowed methods for that handler
  */
 const ALLOWED_METHODS = ['POST'];
 

src/code/handler.js

@@ -12,7 +12,7 @@
 import { Response } from '@adobe/fetch';
 
 /**
- * Allowed methods for that handler.
+ * Allowed methods for that handler
  */
 const ALLOWED_METHODS = ['POST'];
 

src/config/utils.js

@@ -107,3 +107,18 @@ export async function getUserListPaths(context) {
   }
   return Array.from(paths);
 }
+
+/**
+ * Return the contents of the `.hlx.json` file in a project.
+ *
+ * @param {import('@adobe/helix-shared-storage').Bucket} contentBus content bus bucket
+ * @param {string} contentBusId content bus id
+ * @returns contents of `.hlx.json` or null
+ */
+export async function fetchHlxJson(contentBus, contentBusId) {
+  const buf = await contentBus.get(`${contentBusId}/.hlx.json`);
+  if (!buf) {
+    return null;
+  }
+  return JSON.parse(buf.toString());
+}

src/contentproxy/handler.js

@@ -13,7 +13,7 @@ import { Response } from '@adobe/fetch';
 import { contentProxy } from './index.js';
 
 /**
- * Allowed methods for that handler.
+ * Allowed methods for that handler
  */
 const ALLOWED_METHODS = ['GET'];
 

src/discover/cdn-identifier.js

@@ -59,7 +59,7 @@ export async function querySiblingSites(context, info) {
   const { owner, repo } = info;
   const codeBusId = `${owner}/${repo}`;
 
-  const inventory = new Inventory(log, HelixStorage.fromContext(context).contentBus());
+  const inventory = new Inventory(HelixStorage.fromContext(context).contentBus(), log);
   if (!await inventory.load()) {
     log.warn('Inventory not available');
     return [];

src/discover/handler.js

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { Response } from '@adobe/fetch';
+import query from './query.js';
+import reindex from './reindex.js';
+import remove from './remove.js';
+
+/**
+ * Allowed methods for that handler
+ */
+const ALLOWED_METHODS = ['GET', 'POST', 'DELETE'];
+
+/**
+ * Handles the discover route.
+ *
+ * @param {import('../support/AdminContext').AdminContext} context context
+ * @param {import('../support/RequestInfo').RequestInfo} info request info
+ * @returns {Promise<Response>} response
+ */
+export default async function discoverHandler(context, info) {
+  const { attributes: { authInfo } } = context;
+
+  if (ALLOWED_METHODS.indexOf(info.method) < 0) {
+    return new Response('method not allowed', {
+      status: 405,
+    });
+  }
+  if (info.method === 'GET') {
+    return query(context, info);
+  }
+
+  authInfo.assertPermissions('discover:write');
+  if (info.method === 'POST') {
+    return reindex(context, info);
+  }
+  return remove(context, info);
+}

src/discover/inventory.js

@@ -34,7 +34,7 @@ import { isDeepStrictEqual } from 'util';
 /**
  * Inventory path in content bus.
  */
-const INVENTORY_PATH = '/default/inventory-v2.json';
+const INVENTORY_PATH = '/default/inventory.json';
 
 /**
  * Simple inventory class
@@ -49,12 +49,12 @@ export class Inventory {
   };
 
   /**
-   * @type {Bucket}
+   * @type {import('@adobe/helix-shared-storage').Bucket}
    */
   #bucket;
 
   /**
-   * @type {Logger}
+   * @type {any}
    */
   #log;
 
@@ -64,10 +64,11 @@ export class Inventory {
   #modified;
 
   /**
-   * @param {Logger} log
-   * @param {Bucket} bucket
+   * @constructs Inventory
+   * @param {import('@adobe/helix-shared-storage').Bucket} bucket bucket
+   * @param {any} log logger
    */
-  constructor(log, bucket) {
+  constructor(bucket, log) {
     this.#log = log;
     this.#bucket = bucket;
     this.#modified = true;

src/discover/matcher/github.js

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Matcher that filters inventory entries against github url.
+ */
+export default class GithubMatcher {
+  /**
+   * Find the inventory entries that have the given github URL.
+   *
+   * @param {URL} url google document or spreadsheet
+   * @param {Inventory} inventory inventory of entries
+   */
+  // eslint-disable-next-line class-methods-use-this
+  filter(url, inventory) {
+    const segs = url.pathname.split('/');
+    const [, owner, repo] = segs;
+    const codeBusId = `${owner}/${repo}`;
+    return inventory.entries().filter((entry) => entry.codeBusId === codeBusId);
+  }
+
+  /**
+   * Test whether this class can handle an URL
+   *
+   * @param {URL} url url to match
+   * @returns true if this class can handle the URL
+   */
+  static match(url) {
+    return url.host === 'github.com';
+  }
+}

src/discover/matcher/google.js

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { getCachePlugin } from '@adobe/helix-shared-tokencache';
+import processQueue from '@adobe/helix-shared-process-queue';
+
+/**
+ * Build an object containing all gdrive root ids as keys.
+ *
+ * @param {array} inventory inventory of repositories
+ * @returns object with gdrive root ids
+ */
+function buildRoots(inventory) {
+  return inventory.filter(({ gdriveId }) => !!gdriveId)
+    .reduce((roots, { gdriveId }) => {
+      // eslint-disable-next-line no-param-reassign
+      roots[gdriveId] = '/';
+      return roots;
+    }, {});
+}
+
+/**
+ * A custom user consists of a project (org/site) and a content bus id.
+ *
+ * @typedef CustomUser
+ * @property {string} project
+ * @property {string} contentBusId
+ */
+
+/**
+ * Matcher that filters inventory entries against known google drives.
+ */
+export default class GoogleMatcher {
+  constructor(context) {
+    this.context = context;
+  }
+
+  /**
+   * Return all custom users that we should use to lookup Google items.
+   *
+   * @param {import('../inventory.js').InventoryEntry[]} entries entries
+   * @returns {CustomUser[]}
+   */
+  #getCustomUsers(entries) {
+    const { env } = this.context;
+
+    return (env.HLX_CUSTOM_GOOGLE_USERS ?? '').split(',')
+      .map((project) => {
+        const [org, site] = project.trim().split('/');
+        return { org, site };
+      })
+      .reduce((users, { org, site }) => {
+        // for orgs (i.e. site = '*'), return just the first custom user
+        // adorned project in that org. this avoids doing a lookup with
+        // the same registered user multiple times
+        const entry = entries.find((e) => !!e.customUser
+          && e.org === org && (site === '*' || e.site === site));
+        if (entry) {
+          const { contentBusId } = entry;
+          users.push({ project: `${org}/${entry.site}`, contentBusId });
+        }
+        return users;
+      }, []);
+  }
+
+  /**
+   * Find the inventory entries that have the given google document, spreadsheet
+   * or folder in their tree.
+   *
+   * @param {URL} url google document or spreadsheet
+   * @param {Inventory} inventory inventory of entries
+   */
+  async filter(url, inventory) {
+    const { log } = this.context;
+
+    const segs = url.pathname.split('/');
+    let id = segs.pop();
+    if (id.startsWith('edit')) {
+      id = segs.pop();
+    }
+    if (id === '') {
+      log.info(`Google URL contains no id: ${url}`);
+      return [];
+    }
+
+    // finding the inventory items for google is a bit more tricky, as we can't match the url with
+    // the mountpoint, because everything is just an ID. we need to lookup the hierarchy of the
+    // item in the url; but for that we need to use the correct connected user. fortunately,
+    // 99% of the projects use the default google user, so we try to resolve with that first.
+    // if the item specified in the url id is not found, we need to traverse all google entries
+    // with the `customUser` flag and try to load it using the entry user.
+
+    try {
+      const entries = inventory.entries();
+
+      // trivial case, id == mountpoint
+      let ret = entries.filter(({ gdriveId }) => gdriveId === id);
+      if (ret.length) {
+        // we don't want to support overlapping projects, so we return the once found here
+        log.info('%j', {
+          discover: {
+            id,
+            count: ret.length,
+            client: false,
+          },
+        });
+        return ret;
+      }
+
+      // resolve using the default user
+      const roots = buildRoots(entries);
+      let client = await this.context.getGoogleClient();
+      let hierarchy = await client.getItemsFromId(id, roots);
+      if (hierarchy.length) {
+        const { id: rootId } = hierarchy[hierarchy.length - 1];
+        ret = entries.filter(({ gdriveId }) => gdriveId === rootId);
+        log.info('%j', {
+          discover: {
+            id,
+            count: ret.length,
+            client: true,
+          },
+        });
+        return ret;
+      }
+
+      // if still nothing found. find using the entries with a custom user
+      ret = null;
+      const customUsers = this.#getCustomUsers(entries);
+      await processQueue(customUsers, async ({ project, contentBusId }) => {
+        if (!ret) {
+          try {
+            // eslint-disable-next-line no-await-in-loop
+            client = await this.context.getGoogleClient(contentBusId);
+            // eslint-disable-next-line no-await-in-loop
+            hierarchy = await client.getItemsFromId(id, roots);
+            if (hierarchy.length) {
+              const { id: rootId } = hierarchy[hierarchy.length - 1];
+              ret = entries.filter(({ gdriveId }) => gdriveId === rootId);
+              log.info('%j', {
+                discover: {
+                  id,
+                  count: ret.length,
+                  client: true,
+                  project,
+                },
+              });
+            }
+          } catch (e) {
+            log.info(`Unable to get items from id: ${url} in ${project}: ${e.message}`);
+          }
+        }
+      }, 3);
+      return ret ?? [];
+    } catch (e) {
+      log.info(`Unable to get items from id: ${url}: ${e.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Test whether this class can handle an URL
+   *
+   * @param {URL} url url to match
+   * @param {Inventory} inventory
+   * @returns true if this class can handle the URL
+   */
+  static match(url, inventory) {
+    return inventory.getHostType(url.hostname) === 'google' || url.hostname.match(/^.*\.google\.com$/);
+  }
+
+  /**
+   * Extract some data from a URL to store in the inventory.
+   *
+   * @param {import('../../index.js').AdminContext} context context
+   * @param {URL} url url to extract data from
+   * @returns object that contains additional entries to store in inventory
+   */
+  static async extract(context, url, entry) {
+    const match = url.pathname.match(/\/.*\/folders\/([^?/]+)$/);
+    if (match) {
+      // eslint-disable-next-line no-param-reassign
+      [, entry.gdriveId] = match;
+      // check for custom user
+      if (entry.contentBusId) {
+        const { code: codeBucket, content: contentBucket } = context.attributes.bucketMap;
+        const plugin = await getCachePlugin({
+          log: context.log,
+          contentBusId: entry.contentBusId,
+          readOnly: true,
+          codeBucket,
+          contentBucket,
+        }, 'google');
+        if (!plugin.key.startsWith('default/.helix-auth/')) {
+          // eslint-disable-next-line no-param-reassign
+          entry.customUser = true;
+        }
+      }
+    }
+  }
+}