feat(registry): basic implementation

packages/core/src/client.ts

@@ -52,8 +52,8 @@ export class Client {
 
   refresh() {
     Object.keys(this.ctx.root[Context.internal]).forEach(async (name) => {
-      if (!name.startsWith('console.')) return
-      const key = name.slice(8)
+      if (!name.startsWith('console.services.')) return
+      const key = name.slice(17)
       const service = this.ctx.get(name) as DataService
       if (!service) return
       if (await this.ctx.serial('console/intercept', this, service.options)) {

packages/core/src/service.ts

@@ -9,7 +9,6 @@ export namespace DataService {
 }
 
 export abstract class DataService<T = never> extends Service {
-  static filter = false
   static inject = ['console']
 
   public async get(forced?: boolean, client?: Client): Promise<T> {

packages/registry/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@cordisjs/registry",
+  "description": "Scan Package Manager for Koishi Plugins",
+  "version": "7.0.3",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "author": "Shigma <shigma10826@gmail.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cordisjs/webui.git",
+    "directory": "packages/registry"
+  },
+  "bugs": {
+    "url": "https://github.com/cordisjs/webui/issues"
+  },
+  "keywords": [
+    "market",
+    "registry",
+    "npm",
+    "package",
+    "search"
+  ],
+  "devDependencies": {
+    "@types/semver": "^7.5.8"
+  },
+  "dependencies": {
+    "cosmokit": "^1.6.2",
+    "p-map": "^4.0.0",
+    "semver": "^7.6.0"
+  }
+}

packages/registry/src/index.ts

@@ -0,0 +1,160 @@
+import { compare, intersects } from 'semver'
+import { Awaitable, defineProperty, Dict, Time } from 'cosmokit'
+import { Registry, RemotePackage, SearchObject, SearchResult } from './types'
+import { conclude } from './utils'
+import pMap from 'p-map'
+
+export * from './local'
+export * from './types'
+export * from './utils'
+
+export interface CollectConfig {
+  step?: number
+  margin?: number
+  timeout?: number
+  ignored?: string[]
+  endpoint?: string
+}
+
+export interface AnalyzeConfig {
+  version: string
+  concurrency?: number
+  before?(object: SearchObject): void
+  onRegistry?(registry: Registry, versions: RemotePackage[]): Awaitable<void>
+  onSuccess?(object: SearchObject, versions: RemotePackage[]): Awaitable<void>
+  onFailure?(name: string, reason: any): Awaitable<void>
+  onSkipped?(name: string): Awaitable<void>
+  after?(object: SearchObject): void
+}
+
+export interface ScanConfig extends CollectConfig, AnalyzeConfig {
+  request<T>(url: string): Promise<T>
+}
+
+const stopWords = [
+  'cordis',
+  'plugin',
+]
+
+export interface RequestConfig {
+  timeout?: number
+}
+
+export default interface Scanner extends SearchResult {
+  progress: number
+}
+
+export default class Scanner {
+  private cache: Dict<SearchObject>
+
+  constructor(public request: <T>(url: string, config?: RequestConfig) => Promise<T>) {
+    defineProperty(this, 'progress', 0)
+    defineProperty(this, 'cache', {})
+  }
+
+  private async search(offset: number, config: CollectConfig) {
+    const { step = 250, timeout = Time.second * 30 } = config
+    const result = await this.request<SearchResult>(`/-/v1/search?text=cordis+plugin&size=${step}&from=${offset}`, { timeout })
+    this.version = result.version
+    for (const object of result.objects) {
+      this.cache[object.package.name] = object
+    }
+    return result.total
+  }
+
+  public async collect(config: CollectConfig = {}) {
+    const { step = 250, margin = 25, ignored = [] } = config
+    this.cache = {}
+    this.time = new Date().toUTCString()
+    const total = await this.search(0, config)
+    for (let offset = Object.values(this.cache).length; offset < total; offset += step - margin) {
+      await this.search(offset - margin, config)
+    }
+    this.objects = Object.values(this.cache).filter((object) => {
+      const { name, date } = object.package
+      // `date` can be `undefined` due to a bug in https://registry.npmjs.org
+      return date && !object.ignored && !ignored.includes(name) && Scanner.isPlugin(name)
+    })
+    this.total = this.objects.length
+  }
+
+  static isPlugin(name: string) {
+    const official = /^@cordisjs\/plugin-[0-9a-z-]+$/.test(name)
+    const community = /(^|\/)cordis-plugin-[0-9a-z-]+$/.test(name)
+    return official || community
+  }
+
+  static isCompatible(range: string, remote: Pick<RemotePackage, 'peerDependencies'>) {
+    const { peerDependencies = {} } = remote
+    const declaredVersion = peerDependencies['cordis']
+    try {
+      return declaredVersion && intersects(range, declaredVersion)
+    } catch {}
+  }
+
+  public async process(object: SearchObject, range: string, onRegistry: AnalyzeConfig['onRegistry']) {
+    const { name } = object.package
+    const official = name.startsWith('@cordisjs/plugin-')
+    const registry = await this.request<Registry>(`/${name}`)
+    const compatible = Object.values(registry.versions).filter((remote) => {
+      return Scanner.isCompatible(range, remote)
+    }).sort((a, b) => compare(b.version, a.version))
+
+    await onRegistry?.(registry, compatible)
+    const versions = compatible.filter(item => !item.deprecated)
+    if (!versions.length) return
+
+    const latest = registry.versions[versions[0].version]
+    const manifest = conclude(latest)
+    const times = compatible.map(item => registry.time[item.version]).sort()
+
+    object.shortname = name.replace(/(cordis-|^@cordisjs\/)plugin-/, '')
+    object.verified = official
+    object.manifest = manifest
+    object.insecure = manifest.insecure
+    object.category = manifest.category
+    object.createdAt = times[0]
+    object.updatedAt = times[times.length - 1]
+    object.package.contributors ??= latest.author ? [latest.author] : []
+    object.package.keywords = (latest.keywords ?? [])
+      .map(keyword => keyword.toLowerCase())
+      .filter((keyword) => {
+        return !keyword.includes(':')
+          && !object.shortname.includes(keyword)
+          && !stopWords.some(word => keyword.includes(word))
+      })
+    return versions
+  }
+
+  public async analyze(config: AnalyzeConfig) {
+    const { concurrency = 5, version, before, onSuccess, onFailure, onSkipped, onRegistry, after } = config
+
+    const result = await pMap(this.objects, async (object) => {
+      if (object.ignored) return
+      before?.(object)
+      const { name } = object.package
+      try {
+        const versions = await this.process(object, version, onRegistry)
+        if (versions) {
+          await onSuccess?.(object, versions)
+          return versions
+        } else {
+          object.ignored = true
+          await onSkipped?.(name)
+        }
+      } catch (error) {
+        object.ignored = true
+        await onFailure?.(name, error)
+      } finally {
+        this.progress += 1
+        after?.(object)
+      }
+    }, { concurrency })
+
+    return result.filter(isNonNullable)
+  }
+}
+
+function isNonNullable<T>(value: T): value is Exclude<T, null | undefined | void> {
+  return value !== null && value !== undefined
+}

packages/registry/src/local.ts

@@ -0,0 +1,88 @@
+/// <reference types="@types/node" />
+
+import { defineProperty, Dict, pick } from 'cosmokit'
+import { dirname } from 'path'
+import { readdir, readFile } from 'fs/promises'
+import { PackageJson, SearchObject, SearchResult } from './types'
+import { conclude } from './utils'
+
+export interface LocalScanner extends SearchResult {}
+
+export class LocalScanner {
+  private cache: Dict<Promise<SearchObject>>
+  private task: Promise<SearchObject[]>
+
+  constructor(public baseDir: string) {
+    defineProperty(this, 'cache', {})
+  }
+
+  onError(reason: any, name: string) {}
+
+  async _collect() {
+    this.cache = {}
+    let root = this.baseDir
+    const tasks: Promise<void>[] = []
+    while (1) {
+      tasks.push(this.loadDirectory(root))
+      const parent = dirname(root)
+      if (root === parent) break
+      root = parent
+    }
+    await Promise.all(tasks)
+    return Promise.all(Object.values(this.cache))
+  }
+
+  async collect(forced = false) {
+    if (forced) delete this.task
+    this.objects = await (this.task ||= this._collect())
+  }
+
+  private async loadDirectory(baseDir: string) {
+    const base = baseDir + '/node_modules'
+    const files = await readdir(base).catch(() => [])
+    for (const name of files) {
+      if (name.startsWith('cordis-plugin-')) {
+        this.cache[name] ||= this.loadPackage(name)
+      } else if (name.startsWith('@')) {
+        const base2 = base + '/' + name
+        const files = await readdir(base2).catch(() => [])
+        for (const name2 of files) {
+          if (name === '@cordisjs' && name2.startsWith('plugin-') || name2.startsWith('cordis-plugin-')) {
+            this.cache[name + '/' + name2] ||= this.loadPackage(name + '/' + name2)
+          }
+        }
+      }
+    }
+  }
+
+  private async loadPackage(name: string) {
+    try {
+      return await this.parsePackage(name)
+    } catch (error) {
+      this.onError(error, name)
+    }
+  }
+
+  private async loadManifest(name: string) {
+    const filename = require.resolve(name + '/package.json')
+    const meta: PackageJson = JSON.parse(await readFile(filename, 'utf8'))
+    meta.peerDependencies ||= {}
+    meta.peerDependenciesMeta ||= {}
+    return [meta, !filename.includes('node_modules')] as const
+  }
+
+  protected async parsePackage(name: string) {
+    const [data, workspace] = await this.loadManifest(name)
+    return {
+      workspace,
+      manifest: conclude(data),
+      shortname: data.name.replace(/(cordis-|^@cordisjs\/)plugin-/, ''),
+      package: pick(data, [
+        'name',
+        'version',
+        'peerDependencies',
+        'peerDependenciesMeta',
+      ]),
+    } as SearchObject
+  }
+}