diff --git a/.vscode/launch.json b/.vscode/launch.json index b9ec4d40fc65d..cfc740f2873bc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -117,4 +117,4 @@ "webRoot": "${workspaceRoot}" } ] -} +} \ No newline at end of file diff --git a/configs/tslint.json b/configs/tslint.json index 579005bcbf441..3af53a7e645b4 100644 --- a/configs/tslint.json +++ b/configs/tslint.json @@ -3,7 +3,7 @@ "rules": { "file-header": [ true, - "[\n\r]+ \\* Copyright \\(C\\) \\d{4} .*[\n\r]+" + "[\n\r]+ \\* Copyright \\(C\\) \\d{4}(-\\d{4})? .*[\n\r]+" ], "class-name": true, "comment-format": [ @@ -64,4 +64,4 @@ "check-type" ] } -} \ No newline at end of file +} diff --git a/examples/browser/package.json b/examples/browser/package.json index f2916020dc3d6..abd3f0ed8ac21 100644 --- a/examples/browser/package.json +++ b/examples/browser/package.json @@ -22,6 +22,7 @@ "@theia/preferences": "^0.3.2", "@theia/process": "^0.3.2", "@theia/python": "^0.3.2", + "@theia/search-in-workspace": "^0.3.2", "@theia/task": "^0.3.2", "@theia/terminal": "^0.3.2", "@theia/typescript": "^0.3.2", diff --git a/examples/electron/package.json b/examples/electron/package.json index 4305ee172a3d5..645ea6b5604be 100644 --- a/examples/electron/package.json +++ b/examples/electron/package.json @@ -25,6 +25,7 @@ "@theia/preferences": "^0.3.2", "@theia/process": "^0.3.2", "@theia/python": "^0.3.2", + "@theia/search-in-workspace": "^0.3.2", "@theia/task": "^0.3.2", "@theia/terminal": "^0.3.2", "@theia/typescript": "^0.3.2", @@ -45,4 +46,4 @@ "devDependencies": { "@theia/cli": "^0.3.2" } -} \ No newline at end of file +} diff --git a/packages/search-in-workspace/compile.tsconfig.json b/packages/search-in-workspace/compile.tsconfig.json new file mode 100644 index 0000000000000..b8b72b49c8822 --- /dev/null +++ b/packages/search-in-workspace/compile.tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../configs/base.tsconfig", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ] +} \ No newline at end of file diff --git a/packages/search-in-workspace/package.json b/packages/search-in-workspace/package.json new file mode 100644 index 0000000000000..29a2e4da161f4 --- /dev/null +++ b/packages/search-in-workspace/package.json @@ -0,0 +1,46 @@ +{ + "name": "@theia/search-in-workspace", + "version": "0.3.2", + "description": "Theia - Search in workspace", + "dependencies": { + "@theia/core": "^0.3.2", + "@theia/editor": "^0.3.2", + "vscode-ripgrep": "^0.7.1-patch.0" + }, + "publishConfig": { + "access": "public" + }, + "theiaExtensions": [ + { + "frontend": "lib/browser/search-in-workspace-frontend-module", + "backend": "lib/node/search-in-workspace-backend-module" + } + ], + "keywords": [ + "theia-extension" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/theia-ide/theia.git" + }, + "bugs": { + "url": "https://github.com/theia-ide/theia/issues" + }, + "homepage": "https://github.com/theia-ide/theia", + "files": [ + "lib", + "src" + ], + "scripts": { + "prepare": "yarn run clean && yarn run build", + "clean": "theiaext clean", + "build": "theiaext build", + "watch": "theiaext watch", + "test": "theiaext test", + "docs": "theiaext docs" + }, + "devDependencies": { + "@theia/ext-scripts": "^0.2.0" + } +} diff --git a/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts b/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts new file mode 100644 index 0000000000000..f38daa08e15e6 --- /dev/null +++ b/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import URI from '@theia/core/lib/common/uri'; +import { QuickOpenService, QuickOpenModel, QuickOpenItem, QuickOpenItemOptions } from '@theia/core/lib/browser/quick-open/'; +import { injectable, inject } from 'inversify'; +import { MenuModelRegistry, MenuContribution, CommandContribution, CommandRegistry, KeybindingContribution, KeybindingRegistry, ILogger } from '@theia/core'; +import { CommonMenus, QuickOpenMode, OpenerService, open, Highlight, QuickOpenOptions } from '@theia/core/lib/browser'; +import { SearchInWorkspaceService } from './search-in-workspace-service'; +import { SearchInWorkspaceResult, SearchInWorkspaceOptions } from '../common/search-in-workspace-interface'; +import { Range } from '@theia/editor/lib/browser'; +import { LabelProvider } from '@theia/core/lib/browser/label-provider'; + +@injectable() +export class QuickSearchInWorkspace implements QuickOpenModel { + private currentSearchId: number = -1; + protected MAX_RESULTS = 100; + + constructor( + @inject(QuickOpenService) protected readonly quickOpenService: QuickOpenService, + @inject(SearchInWorkspaceService) protected readonly searchInWorkspaceService: SearchInWorkspaceService, + @inject(OpenerService) protected readonly openerService: OpenerService, + @inject(LabelProvider) protected readonly labelProvider: LabelProvider, + @inject(ILogger) protected readonly logger: ILogger, + ) { } + + onType(lookFor: string, acceptor: (items: QuickOpenItem[]) => void): void { + // If we have a search pending, it's not relevant anymore, cancel it. + this.cancelCurrentSeach(); + + if (lookFor.length === 0) { + // The user has emptied the search box, call acceptor to + // remove any previously shown results. + acceptor([]); + return; + } + + // Options passed to the search service. + const opts: SearchInWorkspaceOptions = { + maxResults: this.MAX_RESULTS, + }; + + // The array in which we'll keep accumulating search results. + const items: QuickSearchInWorkspaceResultItem[] = []; + + this.searchInWorkspaceService.search(lookFor, { + + onResult: (searchId: number, result: SearchInWorkspaceResult) => { + // Is this result from a previous search? + if (searchId !== this.currentSearchId) { + return; + } + + items.push(new QuickSearchInWorkspaceResultItem(result, this.openerService, this.labelProvider)); + }, + + onDone: (searchId: number, error?: string) => { + if (searchId !== this.currentSearchId) { + this.logger.debug('Search ' + this.currentSearchId + ' has completed, but it\'s not the current search.'); + return; + } + this.logger.debug('Search ' + this.currentSearchId + ' has completed and is the current search.'); + this.currentSearchId = -1; + + if (error) { + this.showFakeResult(error, acceptor); + } else if (items.length !== 0) { + items.sort((a, b) => SearchInWorkspaceResult.compare(a.getResult(), b.getResult())); + acceptor(items); + } else { + this.showFakeResult('No matches :(', acceptor); + } + + }, + }, opts).then(searchId => { + this.currentSearchId = searchId; + }); + } + + showFakeResult(label: string, acceptor: (items: QuickOpenItem[]) => void) { + acceptor([ + new QuickOpenItem({ + label: label, + }), + ]); + } + + // If we have an ongoing search, cancel it. + cancelCurrentSeach() { + if (this.currentSearchId >= 0) { + this.logger.debug('Cancelling search ' + this.currentSearchId); + this.searchInWorkspaceService.cancel(this.currentSearchId); + this.currentSearchId = -1; + } + } + + // Open the quick search in workspace popup. + open() { + const opts: QuickOpenOptions = { + onClose: cancelled => this.cancelCurrentSeach(), + placeholder: 'Search in workspace by regular expression...', + }; + this.quickOpenService.open(this, opts); + } +} + +class QuickSearchInWorkspaceResultItem extends QuickOpenItem { + + private result: SearchInWorkspaceResult; + private openerService: OpenerService; + + constructor(result: SearchInWorkspaceResult, openerService: OpenerService, labelProvider: LabelProvider) { + const resultHl: Highlight = { + start: result.character - 1, + end: result.character + result.length - 1, + }; + + // Show the path relative to the workspace. + const uri = new URI('file://' + result.file); + const file = labelProvider.getName(uri); + const dir = labelProvider.getLongName(uri.parent) + '/'; + + const filenameHl: Highlight = { + start: 0, + end: file.length, + }; + + const opts: QuickOpenItemOptions = { + detail: result.lineText, + detailHighlights: [resultHl], + label: `${file}:${result.line} - ${dir}`, + labelHighlights: [filenameHl], + }; + super(opts); + + this.result = result; + this.openerService = openerService; + } + + run(mode: QuickOpenMode): boolean { + if (mode !== QuickOpenMode.OPEN) { + return false; + } + + // Search results are 1-based, positions in editors are 0-based. + const line = this.result.line - 1; + const character = this.result.character - 1; + const uri = new URI('file://' + this.result.file); + const r = Range.create(line, character, line, character + this.result.length); + open(this.openerService, uri, { selection: r }); + + return true; + } + + getResult(): SearchInWorkspaceResult { + return this.result; + } +} + +const OpenQuickSearchInWorkspaceCommand = { + id: 'QuickSearchInWorkspace.open', + label: "Search in workspace..." +}; + +@injectable() +export class SearchInWorkspaceContributions implements CommandContribution, MenuContribution, KeybindingContribution { + constructor( + @inject(QuickSearchInWorkspace) protected readonly quickSeachInWorkspace: QuickSearchInWorkspace, + ) { } + + registerCommands(registry: CommandRegistry): void { + registry.registerCommand(OpenQuickSearchInWorkspaceCommand, { + execute: what => this.quickSeachInWorkspace.open() + }); + } + + registerMenus(menus: MenuModelRegistry): void { + menus.registerMenuAction(CommonMenus.EDIT_FIND, { + commandId: OpenQuickSearchInWorkspaceCommand.id, + label: OpenQuickSearchInWorkspaceCommand.label, + }); + } + + registerKeybindings(keybindings: KeybindingRegistry): void { + keybindings.registerKeybinding({ + command: OpenQuickSearchInWorkspaceCommand.id, + keybinding: 'ctrlcmd+shift+f', + }); + } +} diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts new file mode 100644 index 0000000000000..3113508928666 --- /dev/null +++ b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule } from "inversify"; +import { SearchInWorkspaceService, SearchInWorkspaceClientImpl } from './search-in-workspace-service'; +import { SearchInWorkspaceServer } from '../common/search-in-workspace-interface'; +import { WebSocketConnectionProvider } from '@theia/core/lib/browser'; +import { QuickSearchInWorkspace, SearchInWorkspaceContributions } from './quick-search-in-workspace'; +import { CommandContribution, MenuContribution, KeybindingContribution } from "@theia/core"; + +export default new ContainerModule(bind => { + bind(QuickSearchInWorkspace).toSelf().inSingletonScope(); + + bind(CommandContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + bind(MenuContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + bind(KeybindingContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + + // The object that gets notified of search results. + bind(SearchInWorkspaceClientImpl).toSelf().inSingletonScope(); + + bind(SearchInWorkspaceService).toSelf().inSingletonScope(); + + // The object to call methods on the backend. + bind(SearchInWorkspaceServer).toDynamicValue(ctx => { + const client = ctx.container.get(SearchInWorkspaceClientImpl); + return WebSocketConnectionProvider.createProxy(ctx.container, '/search-in-workspace', client); + }).inSingletonScope(); +}); diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-service.ts b/packages/search-in-workspace/src/browser/search-in-workspace-service.ts new file mode 100644 index 0000000000000..c737cae744bf4 --- /dev/null +++ b/packages/search-in-workspace/src/browser/search-in-workspace-service.ts @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { injectable, inject } from "inversify"; +import { SearchInWorkspaceServer, SearchInWorkspaceClient, SearchInWorkspaceResult, SearchInWorkspaceOptions } from "../common/search-in-workspace-interface"; +import { WorkspaceService } from "@theia/workspace/lib/browser"; +import URI from "@theia/core/lib/common/uri"; +import { ILogger } from "@theia/core"; + +/** + * Class that will receive the search results from the server. This is separate + * from the SearchInWorkspaceService class only to avoid a cycle in the + * dependency injection. + */ + +@injectable() +export class SearchInWorkspaceClientImpl implements SearchInWorkspaceClient { + private service: SearchInWorkspaceClient; + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + this.service.onResult(searchId, result); + } + onDone(searchId: number, error?: string): void { + this.service.onDone(searchId, error); + } + + setService(service: SearchInWorkspaceClient) { + this.service = service; + } +} + +export type SearchInWorkspaceCallbacks = SearchInWorkspaceClient; + +/** + * Service to search text in the workspace files. + */ + +@injectable() +export class SearchInWorkspaceService implements SearchInWorkspaceClient { + + // All the searches that we have started, that are not done yet (onDone + // with that searchId has not been called). + private pendingSearches = new Map(); + + // Due to the asynchronicity of the node backend, it's possible that we + // start a search, receive an event for that search, and then receive + // the search id for that search.We therefore need to keep those + // events until we get the search id and return it to the caller. + // Otherwise the caller would discard the event because it doesn't know + // the search id yet. + private pendingOnDones: Map = new Map(); + + private lastKnownSearchId: number = -1; + + constructor( + @inject(SearchInWorkspaceServer) protected readonly searchServer: SearchInWorkspaceServer, + @inject(SearchInWorkspaceClientImpl) protected readonly client: SearchInWorkspaceClientImpl, + @inject(WorkspaceService) protected readonly workspaceService: WorkspaceService, + @inject(ILogger) protected readonly logger: ILogger, + ) { + client.setService(this); + } + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + const callbacks = this.pendingSearches.get(searchId); + + if (callbacks) { + callbacks.onResult(searchId, result); + } + } + + onDone(searchId: number, error?: string): void { + const callbacks = this.pendingSearches.get(searchId); + + if (callbacks) { + this.pendingSearches.delete(searchId); + callbacks.onDone(searchId, error); + } else { + if (searchId > this.lastKnownSearchId) { + this.logger.debug(`Got an onDone for a searchId we don't know about (${searchId}), stashing it for later with error = `, error); + this.pendingOnDones.set(searchId, error); + } else { + // It's possible to receive an onDone for a search we have cancelled. Just ignore it. + this.logger.debug(`Got an onDone for a searchId we don't know about (${searchId}), but it's probably an old one, error = `, error); + } + } + } + + // Start a search of the string "what" in the workspace. + async search(what: string, callbacks: SearchInWorkspaceCallbacks, opts?: SearchInWorkspaceOptions): Promise { + const root = await this.workspaceService.root; + + if (!root) { + throw new Error("Search failed: no workspace root."); + } + + const rootUri = new URI(root.uri); + const searchId = await this.searchServer.search(what, rootUri.path.toString(), opts); + this.pendingSearches.set(searchId, callbacks); + this.lastKnownSearchId = searchId; + + this.logger.debug('Service launched search ' + searchId); + + // Check if we received an onDone before search() returned. + if (this.pendingOnDones.has(searchId)) { + this.logger.debug('Ohh, we have a stashed onDone for that searchId'); + const error = this.pendingOnDones.get(searchId); + this.pendingOnDones.delete(searchId); + + // Call the client's searchId, but first give it a + // chance to record the returned searchId. + setTimeout(() => { + this.onDone(searchId, error); + }, 0); + } + + return searchId; + } + + // Cancel an ongoing search. + cancel(searchId: number) { + this.pendingSearches.delete(searchId); + this.searchServer.cancel(searchId); + } +} diff --git a/packages/search-in-workspace/src/common/search-in-workspace-interface.ts b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts new file mode 100644 index 0000000000000..867737a3c85a8 --- /dev/null +++ b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { JsonRpcServer } from "@theia/core"; + +export interface SearchInWorkspaceOptions { + /** + * Maximum number of results to return. Defaults to unlimited. + */ + maxResults?: number; +} + +export interface SearchInWorkspaceResult { + /** + * The path to the file containing the result. + */ + file: string; + + /** + * The (1-based) line number of the result. + */ + line: number; + + /** + * The (1-based) character number in the result line. For UTF-8 files, + * one multi-byte character counts as one character. + */ + character: number; + + /** + * The length of the match, in characters. For UTF-8 files, one + * multi-byte character counts as one character. + */ + length: number; + + /** + * The text of the line containing the result. + */ + lineText: string; +} + +export namespace SearchInWorkspaceResult { + /** + * Sort search in workspace results according to file, line, character position + * and then length. + */ + export function compare(a: SearchInWorkspaceResult, b: SearchInWorkspaceResult) { + if (a.file !== b.file) { + return a.file < b.file ? -1 : 1; + } + + if (a.line !== b.line) { + return a.line - b.line; + } + + if (a.character !== b.character) { + return a.character - b.character; + } + + return a.length - b.length; + } +} + +export const SearchInWorkspaceClient = Symbol("SearchInWorkspaceClient"); +export interface SearchInWorkspaceClient { + /** + * Called by the server for every search match. + */ + onResult(searchId: number, result: SearchInWorkspaceResult): void; + + /** + * Called when no more search matches will come. + */ + onDone(searchId: number, error?: string): void; +} + +export const SearchInWorkspaceServer = Symbol("SearchInWorkspaceServer"); +export interface SearchInWorkspaceServer extends JsonRpcServer { + /** + * Start a search for WHAT in directory ROOT. Return a unique search id. + */ + search(what: string, root: string, opts?: SearchInWorkspaceOptions): Promise; + + /** + * Cancel an ongoing search. + */ + cancel(searchId: number): Promise; + + dispose(): void; +} diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts new file mode 100644 index 0000000000000..8b15393b7ec6e --- /dev/null +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts @@ -0,0 +1,410 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as chai from 'chai'; +const expect = chai.expect; +import * as temp from 'temp'; +import * as fs from 'fs'; +import { RipgrepSearchInWorkspaceServer } from './ripgrep-search-in-workspace-server'; +import { SearchInWorkspaceClient, SearchInWorkspaceResult } from '../common/search-in-workspace-interface'; +import { Container, ContainerModule } from 'inversify'; +import { ILogger, isWindows } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { RawProcessFactory, RawProcessOptions, RawProcess, ProcessManager } from '@theia/process/lib/node'; +import * as path from 'path'; + +// Allow creating temporary files, but remove them when we are done. +const track = temp.track(); + +// The root dir we'll use to test searching. +let rootDir: string; + +// Remember the content of the test files we create, to validate that the +// reported line text is right. +const fileLines: Map = new Map(); + +// The class under test. +let ripgrepServer: RipgrepSearchInWorkspaceServer; + +// Mock client that accumulates the returned results in a list. +class ResultAccumulator implements SearchInWorkspaceClient { + results: SearchInWorkspaceResult[] = []; + onDoneCallback: () => void; + + constructor(onDoneCallback: () => void) { + this.onDoneCallback = onDoneCallback; + } + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + this.results.push(result); + } + + onDone(searchId: number): void { + // Sort the results, so that the order is predictable. + this.results.sort(SearchInWorkspaceResult.compare); + this.onDoneCallback(); + } +} + +// Create a test file relative to rootDir. +function createTestFile(filename: string, text: string) { + fs.writeFileSync(rootDir + '/' + filename, text); + fileLines.set(filename, text.split('\n')); +} + +before(() => { + rootDir = track.mkdirSync(); + + createTestFile('carrots', `\ +This is a carrot. +Most carrots are orange, but some carrots are not. +Once capitalized, the word carrot looks like this: CARROT. +Carrot is a funny word. +`); + createTestFile('potatoes', `\ +Potatoes, unlike carrots, are generally not orange. But sweet potatoes are, +it's very confusing. +`); + + createTestFile('regexes', `\ +aaa hello x h3lo y hell0h3lllo +hello1 +`); + + fs.mkdirSync(rootDir + '/small'); + createTestFile('small/small', 'A small file.\n'); + + if (!isWindows) { + createTestFile('file:with:some:colons', `\ +Are you looking for this: --foobar? +`); + } + + createTestFile('file with spaces', `\ +Are you looking for this: --foobar? +`); + + createTestFile('utf8-file', `\ +Var är jag? Varför är jag här? +`); + + let lotsOfMatchesText = ''; + for (let i = 0; i < 100000; i++) { + lotsOfMatchesText += 'lots-of-matches\n'; + } + createTestFile('lots-of-matches', lotsOfMatchesText); +}); + +beforeEach(() => { + const container = new Container(); + + const module = new ContainerModule(bind => { + bind(ILogger).to(MockLogger); + bind(RipgrepSearchInWorkspaceServer).toSelf(); + bind(ProcessManager).toSelf().inSingletonScope(); + bind(RawProcess).toSelf().inTransientScope(); + bind(RawProcessFactory).toFactory(ctx => + (options: RawProcessOptions) => { + const child = new Container({ defaultScope: 'Singleton' }); + child.parent = ctx.container; + + child.bind(RawProcessOptions).toConstantValue(options); + return child.get(RawProcess); + } + ); + }); + + container.load(module); + + ripgrepServer = container.get(RipgrepSearchInWorkspaceServer); +}); + +after(() => { + track.cleanupSync(); +}); + +// Compare expected and actual search results. +// +// For convenience, the expected entries do not have their lineText field set +// by individual tests. Using on the file and line fields, this function +// retrieves the expected line text based on what we have written to the test +// files. +// +// The expected entries should also have the file field set relatively to +// rootDir. This function will update the field to contain the absolute path. + +function compareSearchResults(expected: SearchInWorkspaceResult[], actual: SearchInWorkspaceResult[]) { + expect(actual.length).eq(expected.length); + + if (actual.length !== expected.length) { + return; + } + + for (let i = 0; i < actual.length; i++) { + const a = actual[i]; + const e = expected[i]; + + const lines = fileLines.get(e.file); + if (lines) { + const line = lines[e.line - 1]; + e.lineText = line; + e.file = path.join(rootDir, e.file); + + expect(a).deep.eq(e); + } else { + // We don't know this file... + expect.fail(); + } + } +} + +describe('ripgrep-search-in-workspace-server', () => { + // Try some simple patterns with different case. + it('returns 7 results when searching for "carrot"', function (done) { + const pattern = 'carrot'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 2, character: 35, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 3, character: 28, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 3, character: 52, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 4, character: 1, length: pattern.length, lineText: '' }, + { file: 'potatoes', line: 1, character: 18, length: pattern.length, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + it('returns 1 result when searching for "Carrot"', function (done) { + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 4, character: 1, length: 6, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search('Carrot', rootDir); + }); + + it('returns 0 result when searching for "CarroT"', function (done) { + const pattern = 'CarroT'; + + const client = new ResultAccumulator(() => { + compareSearchResults([], client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try something that we know isn't there. + it('finds 0 result when searching for "PINEAPPLE"', function (done) { + const pattern = 'PINEAPPLE'; + + const client = new ResultAccumulator(() => { + compareSearchResults([], client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try a pattern with a space. + it('finds 1 result when searching for "carrots are orange"', function (done) { + const pattern = 'carrots are orange'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with an output size that exceeds the default node buffer size + // (200 * 1024) when spawning a new process. + it('works with a lot of results', function (done) { + // This can take a bit of time. + this.timeout(150000); + const pattern = 'lots-of-matches'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = []; + + for (let i = 1; i <= 100000; i++) { + expected.push({ + file: 'lots-of-matches', + line: i, + character: 1, + length: pattern.length, + lineText: '', + }); + } + + compareSearchResults(expected, client.results); + done(); + }); + + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try limiting the number of returned results. + it('limits the number of returned results', function (done) { + const pattern = 'lots-of-matches'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = []; + + for (let i = 1; i <= 1000; i++) { + expected.push({ + file: 'lots-of-matches', + line: i, + character: 1, + length: pattern.length, + lineText: '', + }); + } + + compareSearchResults(expected, client.results); + done(); + }); + + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir, { + maxResults: 1000, + }); + }); + + // Try with regexes. + it('searches for regexes', function (done) { + const pattern = 'h[e3]l+[o0]'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'regexes', line: 1, character: 5, length: 5, lineText: '' }, + { file: 'regexes', line: 1, character: 13, length: 4, lineText: '' }, + { file: 'regexes', line: 1, character: 20, length: 5, lineText: '' }, + { file: 'regexes', line: 1, character: 25, length: 6, lineText: '' }, + { file: 'regexes', line: 2, character: 1, length: 5, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with a pattern starting with -, and in filenames containing colons and spaces. + it('searches a pattern starting with -', function (done) { + const pattern = '-fo+bar'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'file with spaces', line: 1, character: 28, length: 7, lineText: '' }, + ]; + + if (!isWindows) { + expected.push( + { file: 'file:with:some:colons', line: 1, character: 28, length: 7, lineText: '' } + ); + } + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with a pattern starting with --, and in filenames containing colons and spaces. + it('searches a pattern starting with --', function (done) { + const pattern = '--fo+bar'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'file with spaces', line: 1, character: 27, length: 8, lineText: '' }, + ]; + + if (!isWindows) { + expected.push( + { file: 'file:with:some:colons', line: 1, character: 27, length: 8, lineText: '' } + ); + } + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try searching in an UTF-8 file. + it('searches in a UTF-8 file', function (done) { + const pattern = ' jag'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'utf8-file', line: 1, character: 7, length: 4, lineText: '' }, + { file: 'utf8-file', line: 1, character: 23, length: 4, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try searching a pattern that contains unicode characters. + it('searches a UTF-8 pattern', function (done) { + const pattern = ' h?är'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'utf8-file', line: 1, character: 4, length: 3, lineText: '' }, + { file: 'utf8-file', line: 1, character: 20, length: 3, lineText: '' }, + { file: 'utf8-file', line: 1, character: 27, length: 4, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // A regex that may match an empty string should not return zero-length + // results. Run the test in a directory without big files, because it + // makes rg print all searched lines, which can take a lot of time. + it('doesn\'t return zero-length matches', function (done) { + const pattern = '(hello)?'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir + '/small'); + }); +}); diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts new file mode 100644 index 0000000000000..cc321c2f311b3 --- /dev/null +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { SearchInWorkspaceServer, SearchInWorkspaceOptions, SearchInWorkspaceResult, SearchInWorkspaceClient } from "../common/search-in-workspace-interface"; +import { ILogger } from "@theia/core"; +import { inject, injectable } from "inversify"; +import { RawProcess, RawProcessFactory, RawProcessOptions } from '@theia/process/lib/node'; + +import * as rg from 'vscode-ripgrep'; + +@injectable() +export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { + + // List of ongoing searches, maps search id to a the started rg process. + private ongoingSearches: Map = new Map(); + + // Each incoming search is given a unique id, returned to the client. This is the next id we will assigned. + private nextSearchId: number = 0; + + private client: SearchInWorkspaceClient | undefined; + + // Highlighted red + private readonly FILENAME_START = '^\x1b\\[m\x1b\\[31m'; + private readonly FILENAME_END = '\x1b\\[m:'; + // Highlighted green + private readonly LINE_START = '^\x1b\\[m\x1b\\[32m'; + private readonly LINE_END = '\x1b\\[m:'; + // Highlighted yellow + private readonly CHARACTER_START = '^\x1b\\[m\x1b\\[33m'; + private readonly CHARACTER_END = '\x1b\\[m:'; + // Highlighted blue + private readonly MATCH_START = '\x1b\\[m\x1b\\[34m\x1b\\[1m'; + private readonly MATCH_END = '\x1b\\[m'; + + constructor( + @inject(ILogger) protected readonly logger: ILogger, + @inject(RawProcessFactory) protected readonly rawProcessFactory: RawProcessFactory, + ) { } + + setClient(client: SearchInWorkspaceClient | undefined): void { + this.client = client; + } + + // Search for the string WHAT in directory ROOT. Return the assigned search id. + search(what: string, root: string, opts?: SearchInWorkspaceOptions): Promise { + // Start the rg process. Use --vimgrep to get one result per + // line, --color=always to get color control characters that + // we'll use to parse the lines. + const searchId = this.nextSearchId++; + const processOptions: RawProcessOptions = { + command: rg.rgPath, + args: ["--vimgrep", "-S", "--color=always", + "--colors=path:fg:red", + "--colors=line:fg:green", + "--colors=column:fg:yellow", + "--colors=match:fg:blue", + "-e", what, root], + }; + const process: RawProcess = this.rawProcessFactory(processOptions); + this.ongoingSearches.set(searchId, process); + + process.onError(error => { + // tslint:disable-next-line:no-any + let errorCode = (error as any).code; + + // Try to provide somewhat clearer error messages, if possible. + if (errorCode === 'ENOENT') { + errorCode = 'could not find the ripgrep (rg) binary'; + } else if (errorCode === 'EACCES') { + errorCode = 'could not execute the ripgrep (rg) binary'; + } + + const errorStr = `An error happened while searching (${errorCode}).`; + this.wrapUpSearch(searchId, errorStr); + }); + + // Running counter of results. + let numResults = 0; + + // Buffer to accumulate incoming output. + let databuf: string = ""; + + const lastMatch = { + file: '', + line: 0, + index: 0, + }; + + process.output.on('data', (chunk: string) => { + // We might have already reached the max number of + // results, sent a TERM signal to rg, but we still get + // the data that was already output in the mean time. + // It's not necessary to return early here (the check + // for maxResults below would avoid sending extra + // results), but it avoids doing unnecessary work. + if (opts && opts.maxResults && numResults >= opts.maxResults) { + return; + } + + databuf += chunk; + + while (1) { + // Check if we have a complete line. + const eolIdx = databuf.indexOf('\n'); + if (eolIdx < 0) { + break; + } + + // Get and remove the line from the data buffer. + let lineBuf = databuf.slice(0, eolIdx); + databuf = databuf.slice(eolIdx + 1); + + // Extract the various fields using the ANSI + // control characters for colors as guides. + + // Extract filename (magenta). + const filenameRE = new RegExp(`${this.FILENAME_START}(.+?)${this.FILENAME_END}`); + let match = filenameRE.exec(lineBuf); + if (!match) { + continue; + } + + const filename = match[1]; + lineBuf = lineBuf.slice(match[0].length); + + // Extract line number (green). + const lineRE = new RegExp(`${this.LINE_START}(\\d+)${this.LINE_END}`); + match = lineRE.exec(lineBuf); + if (!match) { + continue; + } + + const line = +match[1]; + lineBuf = lineBuf.slice(match[0].length); + + // Extract character number (column), but don't + // do anything with it. ripgrep reports the + // offset in bytes, which is not good when + // dealing with multi-byte UTF-8 characters. + const characterNumRE = new RegExp(`${this.CHARACTER_START}(\\d+)${this.CHARACTER_END}`); + match = characterNumRE.exec(lineBuf); + if (!match) { + continue; + } + + lineBuf = lineBuf.slice(match[0].length); + + // If there are two matches in a line, + // --vimgrep will make rg output two lines, but + // both matches will be highlighted in both + // lines. If we have consecutive matches at + // the same file / line, make sure to pick the + // right highlighted match. + if (lastMatch.file === filename && lastMatch.line === line) { + lastMatch.index++; + } else { + lastMatch.file = filename; + lastMatch.line = line; + lastMatch.index = 0; + } + + // Extract the match text (red). + const matchRE = new RegExp(`${this.MATCH_START}(.*?)${this.MATCH_END}`); + + let characterNum = 0; + + let matchWeAreLookingFor: RegExpMatchArray | undefined = undefined; + for (let i = 0; ; i++) { + const nextMatch = lineBuf.match(matchRE); + + if (!nextMatch) { + break; + } + + // Just to make typescript happy. + if (nextMatch.index === undefined) { + break; + } + + if (i === lastMatch.index) { + matchWeAreLookingFor = nextMatch; + characterNum = nextMatch.index + 1; + } + + // Remove the control characters around the match. This allows to: + + // - prepare the line text so it can be returned to the client without control characters + // - get the character index of subsequent matches right + + lineBuf = + lineBuf.slice(0, nextMatch.index) + + nextMatch[1] + + lineBuf.slice(nextMatch.index + nextMatch[0].length); + } + + if (!matchWeAreLookingFor || characterNum === 0) { + continue; + } + + if (matchWeAreLookingFor[1].length === 0) { + continue; + } + + const result: SearchInWorkspaceResult = { + file: filename, + line: line, + character: characterNum, + length: matchWeAreLookingFor[1].length, + lineText: lineBuf, + }; + + numResults++; + if (this.client) { + this.client.onResult(searchId, result); + } + + // Did we reach the maximum number of results? + if (opts && opts.maxResults && numResults >= opts.maxResults) { + process.kill(); + this.wrapUpSearch(searchId); + break; + } + } + }); + + process.output.on('end', () => { + // If we reached maxResults, we should have already + // wrapped up the search. Returning early avoids + // logging a warning message in wrapUpSearch. + if (opts && opts.maxResults && numResults >= opts.maxResults) { + return; + } + + this.wrapUpSearch(searchId); + }); + + return Promise.resolve(searchId); + } + + // Cancel an ongoing search. Trying to cancel a search that doesn't exist isn't an + // error, otherwise we'd have to deal with race conditions, where a client cancels a + // search that finishes normally at the same time. + cancel(searchId: number): Promise { + const process = this.ongoingSearches.get(searchId); + if (process) { + process.kill(); + this.wrapUpSearch(searchId); + } + + return Promise.resolve(); + } + + // Send onDone to the client and clean up what we know about search searchId. + private wrapUpSearch(searchId: number, error?: string) { + if (this.ongoingSearches.delete(searchId)) { + if (this.client) { + this.logger.debug("Sending onDone for " + searchId, error); + this.client.onDone(searchId, error); + } else { + this.logger.debug("Wrapping up search " + searchId + " but no client"); + } + } else { + this.logger.debug("Trying to wrap up a search we don't know about " + searchId); + } + } + + dispose(): void { + } +} diff --git a/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts b/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts new file mode 100644 index 0000000000000..a3bf31ca74f73 --- /dev/null +++ b/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule } from "inversify"; +import { ConnectionHandler, JsonRpcConnectionHandler } from "@theia/core/lib/common"; +import { SearchInWorkspaceServer, SearchInWorkspaceClient } from "../common/search-in-workspace-interface"; +import { RipgrepSearchInWorkspaceServer } from "./ripgrep-search-in-workspace-server"; + +export default new ContainerModule(bind => { + bind(SearchInWorkspaceServer).to(RipgrepSearchInWorkspaceServer); + bind(ConnectionHandler).toDynamicValue(ctx => + new JsonRpcConnectionHandler + ('/search-in-workspace', client => { + const server = ctx.container.get(SearchInWorkspaceServer); + server.setClient(client); + client.onDidCloseConnection(() => server.dispose()); + return server; + }) + ); +}); diff --git a/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts b/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts new file mode 100644 index 0000000000000..a48e154d34a3e --- /dev/null +++ b/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts @@ -0,0 +1,3 @@ +declare module 'vscode-ripgrep' { + export const rgPath: string; +}