From 550a38af1f08a7e330846e0681902581ca42ed24 Mon Sep 17 00:00:00 2001 From: Anton Kosiakov Date: Fri, 29 Sep 2017 12:10:21 +0200 Subject: [PATCH] =?UTF-8?q?[extension-manager]=C2=A0the=20extension=20prot?= =?UTF-8?q?ocol=20definition=20and=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anton Kosiakov --- .gitignore | 3 +- .vscode/launch.json | 75 ++++-- .../src/application-package-manager.ts | 18 +- .../src/application-package.ts | 4 +- .../src/application-process.ts | 14 +- .../application-package/src/npm-registry.ts | 7 + examples/browser/package.json | 1 + examples/electron/package.json | 1 + packages/core/src/common/os.ts | 16 +- packages/extension-manager/.gitignore | 1 + packages/extension-manager/README.md | 6 + .../extension-manager/compile.tsconfig.json | 10 + packages/extension-manager/package.json | 53 ++++ .../src/browser/extension-frontend-module.ts | 19 ++ .../src/common/extension-manager.ts | 150 +++++++++++ .../src/common/extension-protocol.ts | 162 ++++++++++++ .../extension-manager/src/common/index.ts | 8 + .../src/node/application-project-cli.ts | 53 ++++ .../src/node/application-project.spec.ts | 93 +++++++ .../src/node/application-project.ts | 210 +++++++++++++++ .../src/node/extension-backend-module.ts | 54 ++++ .../src/node/node-extension-server.spec.ts | 188 +++++++++++++ .../src/node/node-extension-server.ts | 249 ++++++++++++++++++ .../extension-manager/src/node/npm-client.ts | 82 ++++++ packages/extension-manager/src/node/npms.ts | 36 +++ .../test/extension-node-test-container.ts | 29 ++ .../test-resources/testproject/package.json | 7 + packages/tsconfig.json | 74 ------ scripts/lerna.js | 1 - tsconfig.json | 87 ++++++ yarn.lock | 69 +++++ 31 files changed, 1665 insertions(+), 115 deletions(-) create mode 100644 packages/extension-manager/.gitignore create mode 100644 packages/extension-manager/README.md create mode 100644 packages/extension-manager/compile.tsconfig.json create mode 100644 packages/extension-manager/package.json create mode 100644 packages/extension-manager/src/browser/extension-frontend-module.ts create mode 100644 packages/extension-manager/src/common/extension-manager.ts create mode 100644 packages/extension-manager/src/common/extension-protocol.ts create mode 100644 packages/extension-manager/src/common/index.ts create mode 100644 packages/extension-manager/src/node/application-project-cli.ts create mode 100644 packages/extension-manager/src/node/application-project.spec.ts create mode 100644 packages/extension-manager/src/node/application-project.ts create mode 100644 packages/extension-manager/src/node/extension-backend-module.ts create mode 100644 packages/extension-manager/src/node/node-extension-server.spec.ts create mode 100644 packages/extension-manager/src/node/node-extension-server.ts create mode 100644 packages/extension-manager/src/node/npm-client.ts create mode 100644 packages/extension-manager/src/node/npms.ts create mode 100644 packages/extension-manager/src/node/test/extension-node-test-container.ts create mode 100644 packages/extension-manager/test-resources/testproject/package.json delete mode 100644 packages/tsconfig.json create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore index 010e2d5bebde9..4e77fef07deb4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ coverage examples/*/src-gen examples/*/webpack.config.js .browser_modules -**/docs/api \ No newline at end of file +**/docs/api +package-backup.json \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 32e8f258343a4..d9700a29bba5c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,15 +16,24 @@ "protocol": "inspector", "args": [ "--loglevel=debug", - "--hostname=localhost" + "--hostname=localhost", + "--no-cluster", + "--app-project-path=${workspaceRoot}/examples/electron", + "--no-app-auto-install" ], + "env": { + "NODE_ENV": "development" + }, "sourceMaps": true, "outFiles": [ "${workspaceRoot}/examples/electron/src-gen/frontend/electron-main.js", "${workspaceRoot}/examples/electron/src-gen/backend/main.js", "${workspaceRoot}/examples/electron/lib/**/*.js", - "${workspaceRoot}/packages/*/lib/**/*.js" - ] + "${workspaceRoot}/packages/*/lib/**/*.js", + "${workspaceRoot}/dev-packages/*/lib/**/*.js" + ], + "smartStep": true, + "console": "integratedTerminal" }, { "type": "node", @@ -33,14 +42,23 @@ "program": "${workspaceRoot}/examples/browser/src-gen/backend/main.js", "args": [ "--loglevel=debug", - "--port=3000" + "--port=3000", + "--no-cluster", + "--app-project-path=${workspaceRoot}/examples/browser", + "--no-app-auto-install" ], + "env": { + "NODE_ENV": "development" + }, "sourceMaps": true, "outFiles": [ - "${workspaceRoot}/examples/browser/src-gen/backend/main.js", + "${workspaceRoot}/examples/browser/src-gen/backend/*.js", "${workspaceRoot}/examples/browser/lib/**/*.js", - "${workspaceRoot}/packages/*/lib/**/*.js" - ] + "${workspaceRoot}/packages/*/lib/**/*.js", + "${workspaceRoot}/dev-packages/*/lib/**/*.js" + ], + "smartStep": true, + "console": "integratedTerminal" }, { "type": "node", @@ -50,38 +68,55 @@ "args": [ "--loglevel=debug", "--root-dir=${workspaceRoot}/../eclipse.jdt.ls/org.eclipse.jdt.ls.core", - "--port=3000" + "--port=3000", + "--no-cluster", + "--no-app-auto-install" ], + "env": { + "NODE_ENV": "development" + }, "sourceMaps": true, "outFiles": [ - "${workspaceRoot}/examples/browser/src-gen/backend/main.js", + "${workspaceRoot}/examples/browser/src-gen/backend/*.js", "${workspaceRoot}/examples/browser/lib/**/*.js", - "${workspaceRoot}/packages/*/lib/**/*.js" - ] + "${workspaceRoot}/packages/*/lib/**/*.js", + "${workspaceRoot}/dev-packages/*/lib/**/*.js" + ], + "smartStep": true, + "internalConsoleOptions": "openOnSessionStart" }, { "type": "node", - "runtimeArgs": [ - "--inspect" - ], "request": "launch", "protocol": "inspector", "name": "Run Mocha Test", "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", "args": [ - "--timeout", - "999999", - "--colors", "${file}", + "--no-timeouts", + "--colors", "--opts", "${workspaceRoot}/packages/mocha.opts" ], "env": { - "TS_NODE_PROJECT": "${workspaceRoot}/packages/tsconfig.json" + "TS_NODE_PROJECT": "${workspaceRoot}/tsconfig.json" }, "sourceMaps": true, - "internalConsoleOptions": "openOnSessionStart", - "port": 9229 + "smartStep": true, + "console": "integratedTerminal" + }, + { + "type": "node", + "request": "attach", + "name": "Attach by Process ID", + "processId": "${command:PickProcess}", + "sourceMaps": true, + "outFiles": [ + "${workspaceRoot}/packages/*/lib/**/*.js", + "${workspaceRoot}/dev-packages/*/lib/**/*.js" + ], + "smartStep": true, + "internalConsoleOptions": "openOnSessionStart" }, { "name": "Launch Frontend", diff --git a/dev-packages/application-package/src/application-package-manager.ts b/dev-packages/application-package/src/application-package-manager.ts index bafdf1742b8d2..a57e8534c9bf6 100644 --- a/dev-packages/application-package/src/application-package-manager.ts +++ b/dev-packages/application-package/src/application-package-manager.ts @@ -13,17 +13,21 @@ import { ApplicationProcess } from './application-process'; export class ApplicationPackageManager { readonly pck: ApplicationPackage; + /** application process */ + readonly process: ApplicationProcess; + /** manager process */ + protected readonly __process: ApplicationProcess; protected readonly webpack: WebpackGenerator; protected readonly backend: BackendGenerator; protected readonly frontend: FrontendGenerator; - protected readonly appProcess: ApplicationProcess; constructor(options: ApplicationPackageOptions) { this.pck = new ApplicationPackage(options); + this.process = new ApplicationProcess(this.pck, options.projectPath); + this.__process = new ApplicationProcess(this.pck, `${__dirname}/..`); this.webpack = new WebpackGenerator(this.pck); this.backend = new BackendGenerator(this.pck); this.frontend = new FrontendGenerator(this.pck); - this.appProcess = new ApplicationProcess(this.pck); } protected async remove(path: string): Promise { @@ -52,7 +56,7 @@ export class ApplicationPackageManager { async build(args: string[] = []): Promise { await this.generate(); await this.copy(); - return this.appProcess.run('webpack', args); + return this.__process.run('webpack', args); } async start(args: string[] = []): Promise { @@ -66,8 +70,8 @@ export class ApplicationPackageManager { if (!args.some(arg => arg.startsWith('--hostname='))) { args.push('--hostname=localhost'); } - return this.appProcess.bunyan( - this.appProcess.spawnBin('electron', [this.pck.frontend('electron-main.js'), ...args], { + return this.__process.bunyan( + this.__process.spawnBin('electron', [this.pck.frontend('electron-main.js'), ...args], { stdio: [0, 'pipe', 'pipe', 'ipc'] }) ); @@ -77,8 +81,8 @@ export class ApplicationPackageManager { if (!args.some(arg => arg.startsWith('--port='))) { args.push('--port=3000'); } - return this.appProcess.bunyan( - this.appProcess.fork(this.pck.backend('main.js'), args, { + return this.__process.bunyan( + this.__process.fork(this.pck.backend('main.js'), args, { stdio: [0, 'pipe', 'pipe', 'ipc'] }) ); diff --git a/dev-packages/application-package/src/application-package.ts b/dev-packages/application-package/src/application-package.ts index da641e3167de3..7243ad153e3ce 100644 --- a/dev-packages/application-package/src/application-package.ts +++ b/dev-packages/application-package/src/application-package.ts @@ -7,7 +7,7 @@ import * as fs from 'fs'; import * as paths from 'path'; -import { NpmRegistry, NpmRegistryOptions, NodePackage, PublishedNodePackage } from './npm-registry'; +import { NpmRegistry, NpmRegistryOptions, NodePackage, PublishedNodePackage, sortByKey } from './npm-registry'; import { Extension, ExtensionPackage, RawExtensionPackage } from './extension-package'; import writeJsonFile = require('write-json-file'); @@ -243,7 +243,7 @@ export class ApplicationPackage { } else { delete dependencies[name]; } - this.pck.dependencies = dependencies; + this.pck.dependencies = sortByKey(dependencies); return true; } diff --git a/dev-packages/application-package/src/application-process.ts b/dev-packages/application-package/src/application-process.ts index a2c73ef88ea2f..a48b96df19e04 100644 --- a/dev-packages/application-package/src/application-process.ts +++ b/dev-packages/application-package/src/application-process.ts @@ -6,6 +6,7 @@ */ import * as path from 'path'; +import * as fs from 'fs-extra'; import * as cp from 'child_process'; import { ApplicationPackage } from './application-package'; @@ -17,7 +18,8 @@ export class ApplicationProcess { }; constructor( - protected readonly pck: ApplicationPackage + protected readonly pck: ApplicationPackage, + protected readonly binProjectPath: string ) { } spawn(command: string, args?: string[], options?: cp.SpawnOptions): cp.ChildProcess { @@ -27,6 +29,9 @@ export class ApplicationProcess { return cp.fork(modulePath, args, Object.assign({}, this.defaultOptions, options)); } + canRun(command: string): boolean { + return fs.existsSync(this.resolveBin(command)); + } run(command: string, args: string[], options?: cp.SpawnOptions): Promise { const commandProcess = this.spawnBin(command, args, options); return this.promisify(command, commandProcess); @@ -36,11 +41,8 @@ export class ApplicationProcess { return this.spawn(binPath, args, options); } protected resolveBin(command: string): string { - const commandPath = path.resolve(__dirname, '..', 'node_modules', '.bin', command); - if (process.platform === 'win32') { - return commandPath + '.cmd'; - } - return commandPath; + const commandPath = path.resolve(this.binProjectPath, 'node_modules', '.bin', command); + return process.platform === 'win32' ? commandPath + '.cmd' : commandPath; } bunyan(childProcess: cp.ChildProcess): Promise { diff --git a/dev-packages/application-package/src/npm-registry.ts b/dev-packages/application-package/src/npm-registry.ts index 224d609fb42c0..5f0d46a97679a 100644 --- a/dev-packages/application-package/src/npm-registry.ts +++ b/dev-packages/application-package/src/npm-registry.ts @@ -63,6 +63,13 @@ export interface ViewResult { [key: string]: any } +export function sortByKey(object: { [key: string]: any }) { + return Object.keys(object).sort().reduce((sorted, key) => { + sorted[key] = object[key]; + return sorted; + }, {} as { [key: string]: any }); +} + export class NpmRegistryOptions { /** * Default: https://registry.npmjs.org/. diff --git a/examples/browser/package.json b/examples/browser/package.json index 0e317dede4508..73e65cb9b5a09 100644 --- a/examples/browser/package.json +++ b/examples/browser/package.json @@ -6,6 +6,7 @@ "@theia/core": "^0.1.1", "@theia/filesystem": "^0.1.1", "@theia/git": "^0.1.1", + "@theia/extension-manager": "^0.1.1", "@theia/workspace": "^0.1.1", "@theia/preferences": "^0.1.1", "@theia/navigator": "^0.1.1", diff --git a/examples/electron/package.json b/examples/electron/package.json index 740746bc6ed2b..b9912d664bd6b 100644 --- a/examples/electron/package.json +++ b/examples/electron/package.json @@ -9,6 +9,7 @@ "@theia/core": "^0.1.1", "@theia/filesystem": "^0.1.1", "@theia/git": "^0.1.1", + "@theia/extension-manager": "^0.1.1", "@theia/workspace": "^0.1.1", "@theia/preferences": "^0.1.1", "@theia/navigator": "^0.1.1", diff --git a/packages/core/src/common/os.ts b/packages/core/src/common/os.ts index b1a6e47286a20..aa48a175f0a64 100644 --- a/packages/core/src/common/os.ts +++ b/packages/core/src/common/os.ts @@ -5,9 +5,6 @@ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -export const isWindows: boolean = (() => { return is('Windows', 'win32') })(); -export const isOSX: boolean = (() => { return is('Mac', 'darwin') })(); - function is(userAgent: string, platform: string): boolean { if (typeof navigator !== 'undefined') { if (navigator.userAgent && navigator.userAgent.indexOf(userAgent) >= 0) { @@ -18,4 +15,15 @@ function is(userAgent: string, platform: string): boolean { return (process.platform === platform); } return false; -} \ No newline at end of file +} + +export const isWindows = is('Windows', 'win32'); +export const isOSX = is('Mac', 'darwin'); + +export type CMD = [string, string[]]; +export function cmd(command: string, ...args: string[]): CMD { + return [ + isWindows ? 'cmd' : command, + isWindows ? ['/c', command, ...args] : args + ]; +} diff --git a/packages/extension-manager/.gitignore b/packages/extension-manager/.gitignore new file mode 100644 index 0000000000000..de647baf2be53 --- /dev/null +++ b/packages/extension-manager/.gitignore @@ -0,0 +1 @@ +testproject_temp \ No newline at end of file diff --git a/packages/extension-manager/README.md b/packages/extension-manager/README.md new file mode 100644 index 0000000000000..075703c5a6e96 --- /dev/null +++ b/packages/extension-manager/README.md @@ -0,0 +1,6 @@ +# Theia - Extension Manager + +See [here](https://github.com/theia-ide/theia) for a detailed documentation. + +## License +[Apache-2.0](https://github.com/theia-ide/theia/blob/master/LICENSE) \ No newline at end of file diff --git a/packages/extension-manager/compile.tsconfig.json b/packages/extension-manager/compile.tsconfig.json new file mode 100644 index 0000000000000..f42b36183ebc6 --- /dev/null +++ b/packages/extension-manager/compile.tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../base.tsconfig", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ] +} diff --git a/packages/extension-manager/package.json b/packages/extension-manager/package.json new file mode 100644 index 0000000000000..f647f32ccac36 --- /dev/null +++ b/packages/extension-manager/package.json @@ -0,0 +1,53 @@ +{ + "name": "@theia/extension-manager", + "version": "0.1.1", + "description": "Theia - Extension Manager", + "dependencies": { + "@theia/application-package": "^0.1.1", + "@theia/core": "^0.1.1", + "@theia/filesystem": "^0.1.1", + "@types/sanitize-html": "^1.13.31", + "@types/showdown": "^1.7.1", + "sanitize-html": "^1.14.1", + "showdown": "^1.7.4" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "theia-extension" + ], + "theiaExtensions": [ + { + "frontend": "lib/browser/extension-frontend-module", + "backend": "lib/node/extension-backend-module" + } + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/theia-ide/theia.git" + }, + "bugs": { + "url": "https://github.com/theia-ide/theia/issues" + }, + "homepage": "https://github.com/theia-ide/theia", + "files": [ + "lib", + "src" + ], + "scripts": { + "prepare": "yarn run clean && yarn run build", + "clean": "theiaext clean", + "build": "theiaext build", + "watch": "theiaext watch", + "test": "theiaext test", + "docs": "theiaext docs" + }, + "devDependencies": { + "@theia/ext-scripts": "^0.1.1" + }, + "nyc": { + "extends": "../nyc.json" + } +} \ No newline at end of file diff --git a/packages/extension-manager/src/browser/extension-frontend-module.ts b/packages/extension-manager/src/browser/extension-frontend-module.ts new file mode 100644 index 0000000000000..9d176f65c6cfa --- /dev/null +++ b/packages/extension-manager/src/browser/extension-frontend-module.ts @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule } from "inversify"; +import { WebSocketConnectionProvider } from '@theia/core/lib/browser'; +import { ExtensionServer, extensionPath } from '../common/extension-protocol'; +import { ExtensionManager } from '../common'; + +export default new ContainerModule(bind => { + bind(ExtensionServer).toDynamicValue(ctx => { + const provider = ctx.container.get(WebSocketConnectionProvider); + return provider.createProxy(extensionPath); + }).inSingletonScope(); + bind(ExtensionManager).toSelf().inSingletonScope(); +}); diff --git a/packages/extension-manager/src/common/extension-manager.ts b/packages/extension-manager/src/common/extension-manager.ts new file mode 100644 index 0000000000000..f808c45dbeba1 --- /dev/null +++ b/packages/extension-manager/src/common/extension-manager.ts @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { inject, injectable } from "inversify"; +import { Event, Emitter, Disposable, DisposableCollection } from "@theia/core"; +import * as protocol from './extension-protocol'; + +/** + * The extension allows to: + * - access its information from the repository; + * - resolve the detailed information from the repository; + * - test whether it is installed or outdated; + * - install, uninstall and update it. + * + * The user code should access extensions and listen to their changes with the extension manager. + */ +export class Extension extends protocol.Extension { + + constructor( + extension: protocol.Extension, + protected readonly server: protocol.ExtensionServer + ) { + super(); + Object.assign(this, extension); + } + + /** + * Resolve the detailed information. + * + * Resolving can be used to refresh an already resolved extension. + */ + resolve(): Promise { + return this.server.resolve(this.name).then(resolved => + Object.assign(this, resolved) + ); + } + + /** + * Intall the latest version of this extension. + */ + install(): void { + this.server.install(this.name); + } + + /** + * Uninstall the extension. + */ + uninstall(): void { + this.server.uninstall(this.name); + } + + /** + * Update the extension to the latest version. + */ + update(): void { + this.server.update(this.name); + } + +} + +/** + * The resolved extension allows to access its detailed information. + */ +export type ResolvedExtension = Extension & protocol.ResolvedExtension; + +/** + * The extension manager allows to: + * - access installed extensions; + * - look up extensions from the repository; + * - listen to changes of: + * - installed extension; + * - and the installation process. + */ +@injectable() +export class ExtensionManager implements Disposable { + + protected readonly onChangedEmitter = new Emitter(); + protected readonly onWillStartInstallationEmitter = new Emitter(); + protected readonly onDidStopInstallationEmitter = new Emitter(); + protected readonly toDispose = new DisposableCollection(); + + constructor( + @inject(protocol.ExtensionServer) protected readonly server: protocol.ExtensionServer + ) { + this.toDispose.push(server); + this.toDispose.push(this.onChangedEmitter); + this.toDispose.push(this.onWillStartInstallationEmitter); + this.toDispose.push(this.onDidStopInstallationEmitter); + this.server.setClient({ + onDidChange: () => this.fireDidChange(), + onWillStartInstallation: () => this.fireWillStartInstallation(), + onDidStopInstallation: () => this.fireDidStopInstallation(), + }); + } + + dispose() { + this.toDispose.dispose(); + } + + /** + * List installed extensions if the given query is undefined or empty. + * Otherwise look up extensions from the repository matching the given query + * taking into the account installed extensions. + */ + list(param?: protocol.SearchParam): Promise { + return this.server.list(param).then(extensions => + extensions.map(extension => + new Extension(extension, this.server) + ) + ); + } + + /** + * Notify when extensions are installed, uninsalled or updated. + */ + get onDidChange(): Event { + return this.onChangedEmitter.event; + } + + protected fireDidChange(): void { + this.onChangedEmitter.fire(undefined); + } + + /** + * Notiy when the installation process is going to be started. + */ + get onWillStartInstallation(): Event { + return this.onWillStartInstallationEmitter.event; + } + + protected fireWillStartInstallation(): void { + this.onWillStartInstallationEmitter.fire(undefined); + } + + /** + * Notiy when the installation process has been finished. + */ + get onDidStopInstallation(): Event { + return this.onDidStopInstallationEmitter.event; + } + + protected fireDidStopInstallation(): void { + this.onDidStopInstallationEmitter.fire(undefined); + } + +} \ No newline at end of file diff --git a/packages/extension-manager/src/common/extension-protocol.ts b/packages/extension-manager/src/common/extension-protocol.ts new file mode 100644 index 0000000000000..d4ee77d1bf9a5 --- /dev/null +++ b/packages/extension-manager/src/common/extension-protocol.ts @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { JsonRpcServer } from "@theia/core"; + +export const extensionPath = '/services/extensions'; + +/** + * The raw extension information from the repository. + */ +export class RawExtension { + readonly name: string; + readonly version: string; + readonly description: string; + readonly author: string; +} + +/** + * The detailed extension information from the repository. + */ +export class ResolvedRawExtension extends RawExtension { + /** + * The detailed description of the extension in HTML. + */ + readonly documentation: string; +} + +/** + * The extension consists of the raw information and the installation state. + */ +export class Extension extends RawExtension { + /** + * Test whether the extension is installed. + */ + readonly installed: boolean; + /** + * Test whether the extension should be updated. + */ + readonly outdated: boolean; +} + +/** + * The resolved extension allows to access its detailed information. + */ +export type ResolvedExtension = Extension & ResolvedRawExtension; + +/** + * The search param to look up extension from the repository. + */ +export interface SearchParam { + /** + * The query with support for filters and other modifiers, see https://api-docs.npms.io/#api-Search. + * The search is always narrowed with extensino keywords, e.g. `keywords:theia-extension` is always appended by default. + */ + readonly query: string; + /** + * The offset in which to start searching from (max of 5000). + * Default value: 0. + */ + readonly from?: number; + /** + * The total number of results to return (max of 250) + * Default value: 25 + */ + readonly size?: number; +} + +export const ExtensionServer = Symbol('ExtensionServer'); +/** + * The extension server allows to: + * - look up raw extensions from the repository and resolve the detailed information about them; + * - list installed extensions as well as install and uninstall them; + * - list outdated extensions as well as update them; + * - look up extensions from the repository taking into the account installed extensions. + * + * The extension server could start the installation process when an extension is installed, uninstalled or updated. + * The user code should use the extension client to listen to changes of installed extensions and the installation process. + */ +export interface ExtensionServer extends JsonRpcServer { + /** + * Look up raw extensions from the repository matching the given query. + */ + search(param: SearchParam): Promise; + /** + * Resolve the detailed extension information from the repository. + */ + resolveRaw(extension: string): Promise; + + /** + * List installed extensions. + */ + installed(): Promise; + /** + * Install the latest version of the given extension. + */ + install(extension: string): Promise; + /** + * Uninstall the given extension. + */ + uninstall(extension: string): Promise; + + /** + * List outdated extensions which is subset of installed extensions. + */ + outdated(): Promise; + /** + * Update the given extension to the latest version. + */ + update(extension: string): Promise; + + /** + * List installed extensions if the given query is undefined or empty. + * Otherwise look up extensions from the repository matching the given query + * taking into the account installed extensions. + */ + list(param?: SearchParam): Promise; + /** + * Resolve the detailed extension from the repository + * taking into the account installed extensions. + */ + resolve(extension: string): Promise; + + /** + * Schedule the installation process to apply extension changes. + */ + scheduleInstall(): Promise; +} + +/** + * The installation process result. + */ +export interface DidStopInstallationParam { + /** + * Test whether the installation process is failed. + */ + readonly failed: boolean; +} + +export const ExtensionClient = Symbol('ExtensionClient'); +/** + * The extension client allows listening to changes of: + * - installed extensions; + * - the installation process. + */ +export interface ExtensionClient { + /** + * Notify when extensions are installed, uninsalled or updated. + */ + onDidChange(): void; + /** + * Notiy when the installation process is going to be started. + */ + onWillStartInstallation(): void; + /** + * Notiy when the installation process has been finished. + */ + onDidStopInstallation(param: DidStopInstallationParam): void; +} diff --git a/packages/extension-manager/src/common/index.ts b/packages/extension-manager/src/common/index.ts new file mode 100644 index 0000000000000..53321f054d7c2 --- /dev/null +++ b/packages/extension-manager/src/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +export * from './extension-manager'; diff --git a/packages/extension-manager/src/node/application-project-cli.ts b/packages/extension-manager/src/node/application-project-cli.ts new file mode 100644 index 0000000000000..9bfb777cda064 --- /dev/null +++ b/packages/extension-manager/src/node/application-project-cli.ts @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as yargs from 'yargs'; +import { injectable } from 'inversify'; +import { CliContribution } from "@theia/core/lib/node"; +import { ApplicationProjectOptions } from './application-project'; +import { NpmClientOptions } from './npm-client'; + +export type ApplicationProjectArgs = ApplicationProjectOptions & NpmClientOptions; + +const appProjectPath = 'app-project-path'; +const appNpmClient = 'app-npm-client'; +const appAutoInstall = 'app-auto-install'; + +@injectable() +export class ApplicationProjectCliContribution implements CliContribution { + + protected _args: ApplicationProjectArgs; + get args(): ApplicationProjectArgs { + return this._args; + } + + configure(conf: yargs.Argv): void { + conf.option(appProjectPath, { + description: "Sets the application project directory", + default: process.cwd() + }); + conf.option(appNpmClient, { + description: "Sets the application npm client", + choices: ["npm", "yarn"], + default: "yarn" + }); + conf.option(appAutoInstall, { + description: "Sets whether the application should be build on package.json changes", + type: "boolean", + default: true + }); + } + + setArguments(args: yargs.Arguments): void { + this._args = { + projectPath: args[appProjectPath], + npmClient: args[appNpmClient], + autoInstall: args[appAutoInstall] + }; + } + +} diff --git a/packages/extension-manager/src/node/application-project.spec.ts b/packages/extension-manager/src/node/application-project.spec.ts new file mode 100644 index 0000000000000..bb041e4f2ee06 --- /dev/null +++ b/packages/extension-manager/src/node/application-project.spec.ts @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as path from 'path'; +import * as fs from 'fs-extra'; +import * as assert from 'assert'; +import { DidStopInstallationParam } from "../common/extension-protocol"; +import extensionNodeTestContainer from './test/extension-node-test-container'; +import { ApplicationProject } from './application-project'; + +process.on('unhandledRejection', (reason, promise) => { + throw reason; +}); + +let appProject: ApplicationProject; +const appProjectPath = path.resolve(__dirname, '..', '..', 'test-resources', 'testproject_temp'); + +export async function assertInstallation(expectation: { + installed?: string[], + uninstalled?: string[] +}): Promise { + const waitForWillInstall = new Promise(resolve => appProject.onWillInstall(resolve)); + const waitForDidInstall = new Promise(resolve => appProject.onDidInstall(resolve)); + + await waitForWillInstall; + const result = await waitForDidInstall; + + if (expectation.installed) { + for (const extension of expectation.installed) { + assert.equal(true, fs.existsSync(path.resolve(appProjectPath, 'node_modules', extension)), extension + ' is not installed'); + } + } + if (expectation.uninstalled) { + for (const extension of expectation.uninstalled) { + assert.equal(false, fs.existsSync(path.resolve(appProjectPath, 'node_modules', extension)), extension + ' is not uninstalled'); + } + } + assert.equal(true, fs.existsSync(path.resolve(appProjectPath, 'lib', 'bundle.js')), 'the bundle is not generated'); + assert.equal(false, result.failed, 'the installation is failed'); +} + +describe("application-project", function () { + + beforeEach(function () { + this.timeout(50000); + fs.removeSync(appProjectPath); + fs.ensureDirSync(appProjectPath); + appProject = extensionNodeTestContainer({ + projectPath: appProjectPath, + npmClient: 'yarn', + autoInstall: false + }).get(ApplicationProject); + }); + + afterEach(function () { + this.timeout(50000); + appProject.dispose(); + fs.removeSync(appProjectPath); + }); + + it("install", async function () { + this.timeout(1800000); + + await fs.writeJSON(path.resolve(appProjectPath, 'package.json'), { + "private": true, + "dependencies": { + "@theia/core": "0.1.1", + "@theia/filesystem": "0.1.1" + } + }); + appProject.scheduleInstall(); + await assertInstallation({ + installed: ['@theia/core', '@theia/filesystem'] + }); + + await fs.writeJSON(path.resolve(appProjectPath, 'package.json'), { + "private": true, + "dependencies": { + "@theia/core": "0.1.1" + } + }); + appProject.scheduleInstall(); + await assertInstallation({ + installed: ['@theia/core'], + uninstalled: ['@theia/filesystem'] + }); + }); + +}); diff --git a/packages/extension-manager/src/node/application-project.ts b/packages/extension-manager/src/node/application-project.ts new file mode 100644 index 0000000000000..87253e3695e81 --- /dev/null +++ b/packages/extension-manager/src/node/application-project.ts @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as os from 'os'; +import * as paths from 'path'; +import * as fs from 'fs-extra'; +import { injectable, inject } from 'inversify'; +import { ApplicationPackageManager, ApplicationPackageOptions } from '@theia/application-package'; +import { + Disposable, DisposableCollection, Event, Emitter, ILogger, + CancellationTokenSource, CancellationToken, isCancelled, checkCancelled +} from "@theia/core"; +import { FileUri, ServerProcess } from "@theia/core/lib/node"; +import { FileSystemWatcherServer, DidFilesChangedParams } from "@theia/filesystem/lib/common/filesystem-watcher-protocol"; +import { DidStopInstallationParam } from '../common/extension-protocol'; +import { NpmClient } from './npm-client'; + +@injectable() +export class ApplicationProjectOptions extends ApplicationPackageOptions { + readonly autoInstall: boolean; +} + +@injectable() +export class ApplicationProject implements Disposable { + + protected readonly packageUri: string; + protected readonly toDispose = new DisposableCollection(); + protected readonly onChangePackageEmitter = new Emitter(); + protected readonly onWillInstallEmitter = new Emitter(); + protected readonly onDidInstallEmitter = new Emitter(); + + constructor( + @inject(ApplicationProjectOptions) readonly options: ApplicationProjectOptions, + @inject(FileSystemWatcherServer) protected readonly fileSystemWatcher: FileSystemWatcherServer, + @inject(ILogger) protected readonly logger: ILogger, + @inject(NpmClient) protected readonly npmClient: NpmClient, + @inject(ServerProcess) protected readonly serverProcess: ServerProcess + ) { + logger.debug('AppProjectOptions', options); + this.backup(); + this.packageUri = FileUri.create(this.packagePath).toString(); + this.toDispose.push(this.fileSystemWatcher); + this.fileSystemWatcher.setClient({ + onDidFilesChanged: changes => this.onDidFilesChanged(changes) + }); + this.fileSystemWatcher.watchFileChanges(this.packageUri).then(watcher => + this.toDispose.push(Disposable.create(() => + this.fileSystemWatcher.unwatchFileChanges(watcher) + )) + ); + this.toDispose.push(this.onWillInstallEmitter); + this.toDispose.push(this.onDidInstallEmitter); + } + + dispose(): void { + this.toDispose.dispose(); + } + + get onDidChangePackage(): Event { + return this.onChangePackageEmitter.event; + } + protected fireDidChangePackage(): void { + this.onChangePackageEmitter.fire(undefined); + } + protected isPackageChanged(param: DidFilesChangedParams): boolean { + return param.changes.some(change => change.uri === this.packageUri); + } + protected onDidFilesChanged(param: DidFilesChangedParams): void { + if (this.isPackageChanged(param)) { + this.fireDidChangePackage(); + this.autoInstall(); + } + } + + createPackageManager(): ApplicationPackageManager { + return new ApplicationPackageManager(Object.assign({ + log: this.logger.info.bind(this.logger), + error: this.logger.error.bind(this.logger) + }, this.options)); + } + + get onWillInstall(): Event { + return this.onWillInstallEmitter.event; + } + protected fireWillInstall(): void { + this.onWillInstallEmitter.fire(undefined); + } + + get onDidInstall(): Event { + return this.onDidInstallEmitter.event; + } + protected fireDidInstall(params: DidStopInstallationParam = { failed: false }): void { + this.onDidInstallEmitter.fire(params); + } + + protected async autoInstall(): Promise { + if (this.options.autoInstall) { + await this.scheduleInstall(); + } + } + + protected installed: Promise = Promise.resolve(); + protected installationTokenSource = new CancellationTokenSource(); + async scheduleInstall(): Promise { + if (this.installationTokenSource) { + this.installationTokenSource.cancel(); + } + this.installationTokenSource = new CancellationTokenSource(); + const token = this.installationTokenSource.token; + this.installed = this.installed.then(() => this.install(token)); + await this.installed; + } + + protected async install(token?: CancellationToken): Promise { + try { + this.fireWillInstall(); + this.logger.info('Intalling the app...'); + + await this.build(token); + await this.restart(token); + + this.backup(); + this.logger.info('The app installation is finished'); + this.fireDidInstall(); + + this.serverProcess.kill(); + } catch (error) { + if (isCancelled(error)) { + this.logger.info('The app installation is cancelled'); + return; + } + this.logger.error('The app installation is failed' + os.EOL, error); + this.fireDidInstall({ + failed: true + }); + await this.revert(token); + } + } + + protected restart(token?: CancellationToken): Promise { + checkCancelled(token); + return this.serverProcess.restart(); + } + + protected async build(token?: CancellationToken): Promise { + this.logger.info('Installing extensions...'); + await this.prepareBuild(token); + this.logger.info('Extensions are installed'); + + this.logger.info('Building the app...'); + await this.doBuild(token); + this.logger.info('The app is built'); + } + protected prepareBuild(token?: CancellationToken): Promise { + checkCancelled(token); + return this.npmClient.execute(this.options.projectPath, 'install', [], token); + } + protected doBuild(token?: CancellationToken): Promise { + checkCancelled(token); + const manager = this.createPackageManager(); + const scripts = manager.pck.pck.scripts; + if (scripts) { + if ('prepare' in scripts) { + return Promise.resolve(); + } + if ('build' in scripts) { + return this.npmClient.execute(this.options.projectPath, 'build', [], token); + } + } + if (manager.process.canRun('theia')) { + return manager.process.run('theia', ['build']); + } + return manager.build(); + } + + protected backup(): void { + const packagePath = this.packagePath; + if (fs.existsSync(packagePath)) { + fs.copySync(packagePath, this.backupPath); + } + } + protected revert(token?: CancellationToken): void { + checkCancelled(token); + try { + this.logger.info('Reverting the app installation ...'); + const backupPath = this.backupPath; + if (fs.existsSync(backupPath)) { + fs.copySync(backupPath, this.packagePath); + } + } catch (error) { + if (isCancelled(error)) { + this.logger.info('Reverting the app installation is cancelled'); + return; + } + this.logger.error('Reverting the app installation is failed' + os.EOL, error); + } + } + protected get backupPath(): string { + return paths.resolve(this.options.projectPath, 'package-backup.json'); + } + + protected get packagePath(): string { + return paths.resolve(this.options.projectPath, 'package.json'); + } + +} diff --git a/packages/extension-manager/src/node/extension-backend-module.ts b/packages/extension-manager/src/node/extension-backend-module.ts new file mode 100644 index 0000000000000..8d1201f67e5e9 --- /dev/null +++ b/packages/extension-manager/src/node/extension-backend-module.ts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule, interfaces } from "inversify"; +import { ConnectionHandler, JsonRpcConnectionHandler } from '@theia/core'; +import { CliContribution } from '@theia/core/lib/node'; +import { ExtensionServer, ExtensionClient, extensionPath } from "../common/extension-protocol"; +import { ExtensionKeywords, NodeExtensionServer } from './node-extension-server'; +import { ApplicationProject, ApplicationProjectOptions } from './application-project'; +import { NpmClient, NpmClientOptions } from './npm-client'; +import { ApplicationProjectArgs, ApplicationProjectCliContribution } from './application-project-cli'; + +export const extensionKeyword = "theia-extension"; + +export function bindNodeExtensionServer(bind: interfaces.Bind, args?: ApplicationProjectArgs): void { + if (args) { + bind(NpmClientOptions).toConstantValue(args); + bind(ApplicationProjectOptions).toConstantValue(args); + } else { + bind(ApplicationProjectCliContribution).toSelf().inSingletonScope(); + bind(CliContribution).toDynamicValue(ctx => ctx.container.get(ApplicationProjectCliContribution)).inSingletonScope(); + bind(NpmClientOptions).toDynamicValue(ctx => + ctx.container.get(ApplicationProjectCliContribution).args + ).inSingletonScope(); + bind(ApplicationProjectOptions).toDynamicValue(ctx => + ctx.container.get(ApplicationProjectCliContribution).args + ).inSingletonScope(); + } + bind(NpmClient).toSelf().inSingletonScope(); + bind(ApplicationProject).toSelf().inSingletonScope(); + + bind(ExtensionKeywords).toConstantValue([extensionKeyword]); + bind(NodeExtensionServer).toSelf(); + bind(ExtensionServer).toDynamicValue(ctx => + ctx.container.get(NodeExtensionServer) + ); +} + +export default new ContainerModule(bind => { + bindNodeExtensionServer(bind); + + bind(ConnectionHandler).toDynamicValue(ctx => + new JsonRpcConnectionHandler(extensionPath, client => { + const server = ctx.container.get(ExtensionServer); + server.setClient(client); + client.onDidCloseConnection(() => server.dispose()); + return server; + }) + ).inSingletonScope(); +}); diff --git a/packages/extension-manager/src/node/node-extension-server.spec.ts b/packages/extension-manager/src/node/node-extension-server.spec.ts new file mode 100644 index 0000000000000..0b5ba90527ef8 --- /dev/null +++ b/packages/extension-manager/src/node/node-extension-server.spec.ts @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as path from 'path'; +import * as fs from 'fs-extra'; +import * as assert from 'assert'; +import { ExtensionClient, ExtensionServer } from '../common/extension-protocol'; +import extensionNodeTestContainer from './test/extension-node-test-container'; +import { ApplicationProject } from './application-project'; + +process.on('unhandledRejection', (reason, promise) => { + throw reason; +}); + +let appProject: ApplicationProject; +let server: ExtensionServer; +const testProjectPath = path.resolve(__dirname, '..', '..', 'test-resources', 'testproject'); +const appProjectPath = path.resolve(__dirname, '..', '..', 'test-resources', 'testproject_temp'); + +export function waitForDidChange(): Promise { + return new Promise(resolve => { + server.setClient({ + onDidChange: () => resolve() + }); + }); +} + +describe("node-extension-server", function () { + + beforeEach(function () { + this.timeout(50000); + fs.removeSync(appProjectPath); + fs.copySync(testProjectPath, appProjectPath); + const container = extensionNodeTestContainer({ + projectPath: appProjectPath, + npmClient: 'yarn', + autoInstall: false + }); + server = container.get(ExtensionServer); + appProject = container.get(ApplicationProject); + }); + + afterEach(function () { + this.timeout(50000); + server.dispose(); + appProject.dispose(); + fs.removeSync(appProjectPath); + }); + + it("search", function () { + this.timeout(30000); + + return server.search({ + query: "filesystem scope:theia" + }).then(extensions => { + assert.equal(extensions.length, 1, JSON.stringify(extensions, undefined, 2)); + assert.equal(extensions[0].name, '@theia/filesystem'); + }); + }); + + it("installed", function () { + this.timeout(10000); + + return server.installed().then(extensions => { + assert.equal(extensions.length, 2, JSON.stringify(extensions, undefined, 2)); + assert.deepEqual(['@theia/core', '@theia/extension-manager'], extensions.map(e => e.name)); + }); + }); + + it("install", async function () { + this.timeout(10000); + + const before = await server.installed(); + assert.equal(false, before.some(e => e.name === '@theia/filesystem'), JSON.stringify(before, undefined, 2)); + + const onDidChangePackage = waitForDidChange(); + + await server.install("@theia/filesystem"); + + await onDidChangePackage; + return server.installed().then(after => { + assert.equal(true, after.some(e => e.name === '@theia/filesystem'), JSON.stringify(after, undefined, 2)); + }); + }); + + it("uninstall", async function () { + this.timeout(10000); + + const before = await server.installed(); + assert.equal(true, before.some(e => e.name === '@theia/extension-manager'), JSON.stringify(before, undefined, 2)); + + const onDidChangePackage = waitForDidChange(); + + await server.uninstall("@theia/extension-manager"); + + await onDidChangePackage; + return server.installed().then(after => { + assert.equal(false, after.some(e => e.name === '@theia/extension-manager'), JSON.stringify(after, undefined, 2)); + }); + }); + + it("outdated", function () { + this.timeout(10000); + + return server.outdated().then(extensions => { + assert.equal(extensions.length, 1, JSON.stringify(extensions, undefined, 2)); + assert.equal(extensions[0].name, '@theia/core'); + }); + }); + + it("update", async function () { + this.timeout(10000); + + const before = await server.outdated(); + assert.equal(true, before.some(e => e.name === '@theia/core'), JSON.stringify(before, undefined, 2)); + + const onDidChangePackage = waitForDidChange(); + + await server.update("@theia/core"); + + await onDidChangePackage; + return server.outdated().then(after => { + assert.equal(false, after.some(e => e.name === '@theia/core'), JSON.stringify(after, undefined, 2)); + }); + }); + + it("list", function () { + this.timeout(10000); + + return server.list().then(extensions => { + assert.equal(extensions.length, 2, JSON.stringify(extensions, undefined, 2)); + + assert.deepEqual([ + { + name: '@theia/core', + installed: true, + outdated: true + }, + { + name: '@theia/extension-manager', + installed: true, + outdated: false + } + ], extensions.map(e => + Object.assign({}, { + name: e.name, + installed: e.installed, + outdated: e.outdated + }) + )); + }); + }); + + it("list with search", function () { + this.timeout(30000); + + return server.list({ + query: "scope:theia" + }).then(extensions => { + const filtered = extensions.filter(e => ['@theia/core', '@theia/filesystem'].indexOf(e.name) !== -1); + assert.equal(filtered.length, 2, JSON.stringify(filtered, undefined, 2)); + + assert.deepEqual([ + { + name: '@theia/core', + installed: true, + outdated: true + }, + { + name: '@theia/filesystem', + installed: false, + outdated: false + } + ], filtered.map(e => + Object.assign({}, { + name: e.name, + installed: e.installed, + outdated: e.outdated + }) + )); + }); + }); + +}); diff --git a/packages/extension-manager/src/node/node-extension-server.ts b/packages/extension-manager/src/node/node-extension-server.ts new file mode 100644 index 0000000000000..597a4ec6be732 --- /dev/null +++ b/packages/extension-manager/src/node/node-extension-server.ts @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as showdown from 'showdown'; +import * as sanitize from 'sanitize-html'; +import { injectable, inject } from 'inversify'; +import { DisposableCollection } from '@theia/core'; +import { PublishedNodePackage, ExtensionPackage } from '@theia/application-package'; +import { + RawExtension, ResolvedRawExtension, Extension, ResolvedExtension, ExtensionServer, ExtensionClient, SearchParam +} from '../common/extension-protocol'; +import * as npms from './npms'; +import { ApplicationProject } from './application-project'; + +export type ExtensionKeywords = string[]; +export const ExtensionKeywords = Symbol('ExtensionKeyword'); + +@injectable() +export class NodeExtensionServer implements ExtensionServer { + + protected client: ExtensionClient | undefined; + protected readonly toDispose = new DisposableCollection(); + + protected readonly busyExtensions = new Set(); + + constructor( + @inject(ApplicationProject) protected readonly project: ApplicationProject, + @inject(ExtensionKeywords) protected readonly extensionKeywords: ExtensionKeywords + ) { + this.toDispose.push(project.onDidChangePackage(() => + this.notification('onDidChange')() + )); + this.toDispose.push(project.onWillInstall(() => + this.notification('onWillStartInstallation')() + )); + this.toDispose.push(project.onDidInstall(params => + this.notification('onDidStopInstallation')(params) + )); + } + + dispose(): void { + this.toDispose.dispose(); + } + + setClient(client: ExtensionClient | undefined): void { + this.client = client; + } + protected notification(notification: T): ExtensionClient[T] { + if (this.client) { + return this.client[notification]; + } + return () => { }; + } + + async search(param: SearchParam): Promise { + const manager = this.project.createPackageManager(); + const query = this.prepareQuery(param.query); + const packages = await npms.search(query, param.from, param.size); + const extensions = []; + for (const raw of packages) { + if (PublishedNodePackage.is(raw)) { + const extensionPackage = manager.pck.newExtensionPackage(raw); + const extension = this.toRawExtension(extensionPackage); + extensions.push(extension); + } + } + return extensions; + } + protected prepareQuery(query: string): string { + const args = query.split(/\s+/).map(v => v.toLowerCase().trim()).filter(v => !!v); + return [`keywords:${this.extensionKeywords.join(',')}`, ...args].join(' '); + } + + async resolveRaw(extension: string): Promise { + const manager = this.project.createPackageManager(); + const extensionPackage = await manager.pck.findExtensionPackage(extension); + if (!extensionPackage) { + throw new Error('The extension package is not found for ' + extension); + } + return this.toResolvedRawExtension(extensionPackage); + } + + async installed(): Promise { + const manager = this.project.createPackageManager(); + return manager.pck.extensionPackages.map(pck => this.toRawExtension(pck)); + } + async install(extension: string): Promise { + this.setBusy(extension, true); + try { + const manager = this.project.createPackageManager(); + const extensionPackage = await manager.pck.findExtensionPackage(extension); + if (!extensionPackage) { + return; + } + const latestVersion = await extensionPackage.getLatestVersion(); + if (!latestVersion) { + return; + } + if (manager.pck.setDependency(extension, latestVersion)) { + await manager.pck.save(); + } + } finally { + this.setBusy(extension, false); + } + } + + async uninstall(extension: string): Promise { + this.setBusy(extension, true); + try { + const manager = this.project.createPackageManager(); + if (manager.pck.setDependency(extension, undefined)) { + await manager.pck.save(); + } + } finally { + this.setBusy(extension, false); + } + } + + async outdated(): Promise { + const result: RawExtension[] = []; + const promises = []; + const manager = this.project.createPackageManager(); + for (const extensionPackage of manager.pck.extensionPackages) { + promises.push(extensionPackage.isOutdated().then(outdated => { + if (outdated) { + result.push(this.toRawExtension(extensionPackage)); + } + })); + } + await Promise.all(promises); + return result; + } + + async update(extension: string): Promise { + this.setBusy(extension, true); + try { + const manager = this.project.createPackageManager(); + const extensionPackage = manager.pck.getExtensionPackage(extension); + if (!extensionPackage || !extensionPackage.version) { + return; + } + if (!await extensionPackage.isOutdated()) { + return; + } + if (manager.pck.setDependency(extension, extensionPackage.latestVersion)) { + await manager.pck.save(); + } + } finally { + this.setBusy(extension, false); + } + } + + async list(param?: SearchParam): Promise { + const manager = this.project.createPackageManager(); + if (param && param.query) { + const found = await this.search(param); + return Promise.all(found.map(raw => { + const extensionPackage = manager.pck.getExtensionPackage(raw.name); + if (extensionPackage) { + return this.toExtension(extensionPackage); + } + return Object.assign(raw, { + busy: this.isBusy(raw.name), + installed: false, + outdated: false + }); + })); + } + return Promise.all(manager.pck.extensionPackages.map(pck => this.toExtension(pck))); + } + + async resolve(extension: string): Promise { + const manager = await this.project.createPackageManager(); + const extensionPackage = await manager.pck.findExtensionPackage(extension); + if (!extensionPackage) { + throw new Error('The extension package is not found for ' + extension); + } + return this.toResolvedExtension(extensionPackage); + } + + protected async toResolvedExtension(extensionPackage: ExtensionPackage): Promise { + const resolvedRawExtension = await this.toResolvedRawExtension(extensionPackage); + return Object.assign(resolvedRawExtension, { + installed: extensionPackage.installed, + outdated: await extensionPackage.isOutdated(), + busy: this.isBusy(extensionPackage.name) + }); + } + + protected async toResolvedRawExtension(extensionPackage: ExtensionPackage): Promise { + const rawExtension = this.toRawExtension(extensionPackage); + const documentation = await this.compileDocumentation(extensionPackage); + return Object.assign(rawExtension, { + documentation + }); + } + + protected async compileDocumentation(extensionPackage: ExtensionPackage): Promise { + const markdownConverter = new showdown.Converter({ + noHeaderId: true, + strikethrough: true + }); + const readme = await extensionPackage.getReadme(); + const readmeHtml = markdownConverter.makeHtml(readme); + return sanitize(readmeHtml, { + allowedTags: sanitize.defaults.allowedTags.concat(['h1', 'h2', 'img']) + }); + } + + protected async toExtension(extensionPackage: ExtensionPackage): Promise { + const rawExtension = this.toRawExtension(extensionPackage); + return Object.assign(rawExtension, { + installed: extensionPackage.installed, + outdated: await extensionPackage.isOutdated(), + busy: this.isBusy(extensionPackage.name) + }); + } + + protected toRawExtension(extensionPackage: ExtensionPackage): RawExtension { + return { + name: extensionPackage.name, + version: extensionPackage.version, + description: extensionPackage.description, + author: extensionPackage.getAuthor() + }; + } + + protected isBusy(extension: string): boolean { + return this.busyExtensions.has(extension); + } + + protected setBusy(extension: string, busy: boolean): void { + if (busy) { + this.busyExtensions.add(extension); + } else { + this.busyExtensions.delete(extension); + } + this.notification('onDidChange')(); + } + + scheduleInstall(): Promise { + return this.project.scheduleInstall(); + } + +} diff --git a/packages/extension-manager/src/node/npm-client.ts b/packages/extension-manager/src/node/npm-client.ts new file mode 100644 index 0000000000000..0d56cdc3c768f --- /dev/null +++ b/packages/extension-manager/src/node/npm-client.ts @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as cp from 'child_process'; +import { injectable, inject } from 'inversify'; +import { cmd, CMD, ILogger, CancellationToken, checkCancelled, cancelled, Disposable } from "@theia/core"; + +@injectable() +export class NpmClientOptions { + readonly npmClient: 'yarn' | 'npm'; +} + +@injectable() +export class NpmClient { + + constructor( + @inject(NpmClientOptions) protected readonly options: NpmClientOptions, + @inject(ILogger) protected readonly logger: ILogger + ) { } + + execute(projectPath: string, command: string, args: string[], token?: CancellationToken): Promise { + checkCancelled(token); + return new Promise((resolve, reject) => { + const npmProcess = this.spawn(projectPath, command, args); + const disposable = token ? token.onCancellationRequested(() => { + npmProcess.kill('SIGKILL'); + reject(cancelled()); + }) : Disposable.NULL; + + npmProcess.stdout.on('data', data => + this.logger.info(data.toString()) + ); + npmProcess.stderr.on('data', data => + this.logger.error(data.toString()) + ); + + npmProcess.on('close', (code, signal) => { + disposable.dispose(); + + if (code !== 0) { + reject(new Error(`Failed ${command} ${args}, code: ${code}, signal: ${signal}`)); + } else { + resolve(); + } + }); + npmProcess.once('error', err => { + disposable.dispose(); + reject(new Error(`Failed ${command} ${args}, the error: ${err}`)); + }); + }); + } + + spawn(projectPath: string, command: string, args?: string[]): cp.ChildProcess { + const npmCommand = this.npmCommand(command); + return this.doSpawn(projectPath, cmd(this.options.npmClient, npmCommand, ...args)); + } + + protected npmCommand(command: string): string { + if (this.options.npmClient === 'yarn') { + return command; + } + if (command === 'add') { + return 'install'; + } + if (command === 'remove') { + return 'uninstall'; + } + return command; + } + + protected doSpawn(projectPath: string, [command, args]: CMD): cp.ChildProcess { + this.logger.info(projectPath, command, ...args); + return cp.spawn(command, args, { + cwd: projectPath + }); + } + +} diff --git a/packages/extension-manager/src/node/npms.ts b/packages/extension-manager/src/node/npms.ts new file mode 100644 index 0000000000000..0e1a44c5a1215 --- /dev/null +++ b/packages/extension-manager/src/node/npms.ts @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as request from "request"; +import { NodePackage } from '@theia/application-package'; + +export function search(query: string, from?: number, size?: number): Promise { + return new Promise((resolve, reject) => { + let url = 'https://api.npms.io/v2/search?q=' + encodeURIComponent(query); + if (from) { + url += '&from=' + from; + } + if (size) { + url += '&size=' + size; + } + request(url, (error, response, body) => { + if (error) { + reject(error); + // tslint:disable-next-line:no-magic-numbers + } else if (response.statusCode === 200) { + const result = JSON.parse(body) as { + results: { + package: NodePackage + }[] + }; + resolve(result.results.map(v => v.package)); + } else { + reject(new Error(`${response.statusCode}: ${response.statusMessage} for ${url}`)); + } + }); + }); +}; \ No newline at end of file diff --git a/packages/extension-manager/src/node/test/extension-node-test-container.ts b/packages/extension-manager/src/node/test/extension-node-test-container.ts new file mode 100644 index 0000000000000..72a7a06ef172b --- /dev/null +++ b/packages/extension-manager/src/node/test/extension-node-test-container.ts @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2017 TypeFox and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { Container } from "inversify"; +import { ConsoleLoggerServer } from "@theia/core/lib/common/console-logger-server"; +import { ILoggerServer } from "@theia/core/lib/common/logger-protocol"; +import { stubRemoteMasterProcessFactory } from "@theia/core/lib/node"; +import { bindServerProcess } from "@theia/core/lib/node/backend-application-module"; +import { bindLogger } from "@theia/core/lib/node/logger-backend-module"; +import { bindFileSystem, bindFileSystemWatcherServer } from "@theia/filesystem/lib/node/filesystem-backend-module"; +import { ApplicationProjectArgs } from "../application-project-cli"; +import { bindNodeExtensionServer } from '../extension-backend-module'; + +export const extensionNodeTestContainer = (args: ApplicationProjectArgs) => { + const container = new Container(); + const bind = container.bind.bind(container); + bindLogger(bind); + bindServerProcess(bind, stubRemoteMasterProcessFactory); + container.rebind(ILoggerServer).to(ConsoleLoggerServer).inSingletonScope(); + bindFileSystem(bind); + bindFileSystemWatcherServer(bind); + bindNodeExtensionServer(bind, args); + return container; +}; +export default extensionNodeTestContainer; diff --git a/packages/extension-manager/test-resources/testproject/package.json b/packages/extension-manager/test-resources/testproject/package.json new file mode 100644 index 0000000000000..4b024d5cdd24c --- /dev/null +++ b/packages/extension-manager/test-resources/testproject/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "dependencies": { + "@theia/core": "0.1.0", + "@theia/extension-manager": "0.1.0" + } +} \ No newline at end of file diff --git a/packages/tsconfig.json b/packages/tsconfig.json deleted file mode 100644 index c5b44f6a02529..0000000000000 --- a/packages/tsconfig.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "extends": "./base.tsconfig", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@theia/core": [ - "core/src/common" - ], - "@theia/core/lib/*": [ - "core/src/*" - ], - "humane-js": [ - "core/src/typings/humane-js" - ], - "@theia/filesystem/lib/*": [ - "filesystem/src/*" - ], - "mv": [ - "filesystem/src/typings/mv" - ], - "trash": [ - "filesystem/src/typings/trash" - ], - "@theia/navigator/lib/*": [ - "navigator/src/*" - ], - "@theia/preferences-api/lib/*": [ - "preferences-api/src/*" - ], - "@theia/preferences/lib/*": [ - "preferences/src/*" - ], - "@theia/workspace/lib/*": [ - "workspace/src/*" - ], - "@theia/terminal/lib/*": [ - "terminal/src/*" - ], - "@theia/languages/lib/*": [ - "languages/src/*" - ], - "@theia/editor/lib/*": [ - "editor/src/*" - ], - "@theia/monaco/lib/*": [ - "monaco/src/*" - ], - "@theia/cpp/lib/*": [ - "cpp/src/*" - ], - "@theia/java/lib/*": [ - "java/src/*" - ], - "@theia/python/lib/*": [ - "python/src/*" - ], - "@theia/git/lib/*": [ - "git/src/*" - ], - "@theia/go/lib/*": [ - "go/src/*" - ], - "@theia/process/lib*": [ - "process/src/*" - ], - "@theia/markers/lib/*": [ - "markers/src/*" - ] - } - }, - "include": [ - "*/src" - ] -} \ No newline at end of file diff --git a/scripts/lerna.js b/scripts/lerna.js index 252bdfa804d08..40950e1a7797a 100644 --- a/scripts/lerna.js +++ b/scripts/lerna.js @@ -6,7 +6,6 @@ */ // @ts-check const path = require('path'); -const cp = require('child_process'); const lernaPath = path.resolve(__dirname, '..', 'node_modules', 'lerna', 'bin', 'lerna'); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000000000..5dbf5dfab22f1 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,87 @@ +{ + "extends": "./packages/base.tsconfig", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@theia/application-package": [ + "dev-packages/application-package/src" + ], + "@theia/application-package/lib/*": [ + "dev-packages/application-package/src/*" + ], + "@theia/core": [ + "packages/core/src/common" + ], + "@theia/core/lib/*": [ + "packages/core/src/*" + ], + "humane-js": [ + "packages/core/src/typings/humane-js" + ], + "@theia/extension-manager/lib/*": [ + "packages/extension-manager/src/*" + ], + "@theia/filesystem/lib/*": [ + "packages/filesystem/src/*" + ], + "mv": [ + "packages/filesystem/src/typings/mv" + ], + "trash": [ + "packages/filesystem/src/typings/trash" + ], + "@theia/navigator/lib/*": [ + "packages/navigator/src/*" + ], + "@theia/preferences-api": [ + "packages/preferences-api/src/common" + ], + "@theia/preferences-api/lib/*": [ + "packages/preferences-api/src/*" + ], + "@theia/preferences/lib/*": [ + "packages/preferences/src/*" + ], + "@theia/workspace/lib/*": [ + "packages/workspace/src/*" + ], + "@theia/terminal/lib/*": [ + "packages/terminal/src/*" + ], + "@theia/languages/lib/*": [ + "packages/languages/src/*" + ], + "@theia/editor/lib/*": [ + "packages/editor/src/*" + ], + "@theia/monaco/lib/*": [ + "packages/monaco/src/*" + ], + "@theia/cpp/lib/*": [ + "packages/cpp/src/*" + ], + "@theia/java/lib/*": [ + "packages/java/src/*" + ], + "@theia/python/lib/*": [ + "packages/python/src/*" + ], + "@theia/git/lib/*": [ + "git/src/*" + ], + "@theia/go/lib/*": [ + "packages/go/src/*" + ], + "@theia/process/lib*": [ + "packages/process/src/*" + ], + "@theia/markers/lib/*": [ + "packages/markers/src/*" + ] + } + }, + "include": [ + "dev-packages/*/src", + "packages/*/src" + ] +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 381057b2721d9..3f7246ef6a65b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -203,6 +203,10 @@ "@types/form-data" "*" "@types/node" "*" +"@types/sanitize-html@^1.13.31": + version "1.13.31" + resolved "https://registry.yarnpkg.com/@types/sanitize-html/-/sanitize-html-1.13.31.tgz#6d10581c302c5678532041ff5502c96121ef176c" + "@types/semver@^5.4.0": version "5.4.0" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-5.4.0.tgz#f3658535af7f1f502acd6da7daf405ffeb1f7ee4" @@ -221,6 +225,10 @@ "@types/glob" "*" "@types/node" "*" +"@types/showdown@^1.7.1": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@types/showdown/-/showdown-1.7.1.tgz#9ba121fc2b2dcad646040034a5581e82ccecc066" + "@types/sinon@^2.3.5": version "2.3.5" resolved "https://registry.yarnpkg.com/@types/sinon/-/sinon-2.3.5.tgz#68f1e0ac15f2eb6cc682b7af87cd517acc77b589" @@ -1908,10 +1916,38 @@ diffie-hellman@^5.0.0: miller-rabin "^4.0.0" randombytes "^2.0.0" +dom-serializer@0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" + dependencies: + domelementtype "~1.1.1" + entities "~1.1.1" + domain-browser@^1.1.1: version "1.1.7" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc" +domelementtype@1, domelementtype@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.0.tgz#b17aed82e8ab59e52dd9c19b1756e0fc187204c2" + +domelementtype@~1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b" + +domhandler@^2.3.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.1.tgz#892e47000a99be55bbf3774ffea0561d8879c259" + dependencies: + domelementtype "1" + +domutils@^1.5.1: + version "1.6.2" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.6.2.tgz#1958cc0b4c9426e9ed367fb1c8e854891b0fa3ff" + dependencies: + dom-serializer "0" + domelementtype "1" + dot-case@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-2.1.1.tgz#34dcf37f50a8e93c2b3bca8bb7fb9155c7da3bee" @@ -2072,6 +2108,10 @@ enhanced-resolve@^3.3.0: object-assign "^4.0.1" tapable "^0.2.7" +entities@^1.1.1, entities@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" + errno@^0.1.1, errno@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.4.tgz#b896e23a9e5e8ba33871fc996abd3635fc9a1c7d" @@ -3128,6 +3168,17 @@ html-comment-regex@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.1.tgz#668b93776eaae55ebde8f3ad464b307a4963625e" +htmlparser2@^3.9.0: + version "3.9.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338" + dependencies: + domelementtype "^1.3.0" + domhandler "^2.3.0" + domutils "^1.5.1" + entities "^1.1.1" + inherits "^2.0.1" + readable-stream "^2.0.2" + http-errors@1.6.2, http-errors@~1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736" @@ -5582,6 +5633,10 @@ regex-cache@^0.4.2: dependencies: is-equal-shallow "^0.1.3" +regexp-quote@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/regexp-quote/-/regexp-quote-0.0.0.tgz#1e0f4650c862dcbfed54fd42b148e9bb1721fcf2" + regexpu-core@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-1.0.0.tgz#86a763f58ee4d7c2f6b102e4764050de7ed90c6b" @@ -5853,6 +5908,14 @@ samsam@1.x, samsam@^1.1.3: version "1.2.1" resolved "https://registry.yarnpkg.com/samsam/-/samsam-1.2.1.tgz#edd39093a3184370cb859243b2bdf255e7d8ea67" +sanitize-html@^1.14.1: + version "1.14.1" + resolved "https://registry.yarnpkg.com/sanitize-html/-/sanitize-html-1.14.1.tgz#730ffa2249bdf18333effe45b286173c9c5ad0b8" + dependencies: + htmlparser2 "^3.9.0" + regexp-quote "0.0.0" + xtend "^4.0.0" + sax@~1.2.1: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -5977,6 +6040,12 @@ shelljs@^0.7.0: interpret "^1.0.0" rechoir "^0.6.2" +showdown@^1.7.4: + version "1.7.4" + resolved "https://registry.yarnpkg.com/showdown/-/showdown-1.7.4.tgz#6bbc9dd2cdb1e5fdd749ecdadc6a47b856594ae0" + dependencies: + yargs "^8.0.1" + sigmund@~1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"