From ab31b869d5e35fa77485fa8aa48cff82df24acd4 Mon Sep 17 00:00:00 2001 From: Simon Marchi Date: Wed, 13 Dec 2017 15:21:00 -0500 Subject: [PATCH] Add quick search in workspace feature This patch adds a "quick" search in workspace feature. It works using the "quick open" widget. The default keybinding to open it is ctrl-shift-F. The user types in the widget and the search results appear as they type. Selecting a search result opens the file at the location of the file. The backend uses ripgrep to implement the search. There is no way to make ripgrep output a truly unambiguous machine parsable output. The best way that people have found is to enable the color output, and use the ANSI color codes as markers for the fields and matches. The vision in the future would be to have a proper "search in directory/workspace" view, which would show the results more permanently. I think this quick search in workspace feature is still useful in the mean time, easy to implement on the UI side, and lays out the foundation in the backend. Signed-off-by: Simon Marchi --- .vscode/launch.json | 2 +- configs/tslint.json | 4 +- examples/browser/package.json | 1 + examples/electron/package.json | 3 +- .../search-in-workspace/compile.tsconfig.json | 10 + packages/search-in-workspace/package.json | 46 ++ .../src/browser/quick-search-in-workspace.ts | 194 +++++++++ .../search-in-workspace-frontend-module.ts | 32 ++ .../browser/search-in-workspace-service.ts | 129 ++++++ .../common/search-in-workspace-interface.ts | 94 ++++ ...ripgrep-search-in-workspace-server.spec.ts | 410 ++++++++++++++++++ .../ripgrep-search-in-workspace-server.ts | 273 ++++++++++++ .../search-in-workspace-backend-module.ts | 24 + .../src/node/typings/vscode-ripgrep.d.ts | 3 + 14 files changed, 1221 insertions(+), 4 deletions(-) create mode 100644 packages/search-in-workspace/compile.tsconfig.json create mode 100644 packages/search-in-workspace/package.json create mode 100644 packages/search-in-workspace/src/browser/quick-search-in-workspace.ts create mode 100644 packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts create mode 100644 packages/search-in-workspace/src/browser/search-in-workspace-service.ts create mode 100644 packages/search-in-workspace/src/common/search-in-workspace-interface.ts create mode 100644 packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts create mode 100644 packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts create mode 100644 packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts create mode 100644 packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index b9ec4d40fc65d..cfc740f2873bc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -117,4 +117,4 @@ "webRoot": "${workspaceRoot}" } ] -} +} \ No newline at end of file diff --git a/configs/tslint.json b/configs/tslint.json index 579005bcbf441..3af53a7e645b4 100644 --- a/configs/tslint.json +++ b/configs/tslint.json @@ -3,7 +3,7 @@ "rules": { "file-header": [ true, - "[\n\r]+ \\* Copyright \\(C\\) \\d{4} .*[\n\r]+" + "[\n\r]+ \\* Copyright \\(C\\) \\d{4}(-\\d{4})? .*[\n\r]+" ], "class-name": true, "comment-format": [ @@ -64,4 +64,4 @@ "check-type" ] } -} \ No newline at end of file +} diff --git a/examples/browser/package.json b/examples/browser/package.json index f2916020dc3d6..abd3f0ed8ac21 100644 --- a/examples/browser/package.json +++ b/examples/browser/package.json @@ -22,6 +22,7 @@ "@theia/preferences": "^0.3.2", "@theia/process": "^0.3.2", "@theia/python": "^0.3.2", + "@theia/search-in-workspace": "^0.3.2", "@theia/task": "^0.3.2", "@theia/terminal": "^0.3.2", "@theia/typescript": "^0.3.2", diff --git a/examples/electron/package.json b/examples/electron/package.json index 4305ee172a3d5..645ea6b5604be 100644 --- a/examples/electron/package.json +++ b/examples/electron/package.json @@ -25,6 +25,7 @@ "@theia/preferences": "^0.3.2", "@theia/process": "^0.3.2", "@theia/python": "^0.3.2", + "@theia/search-in-workspace": "^0.3.2", "@theia/task": "^0.3.2", "@theia/terminal": "^0.3.2", "@theia/typescript": "^0.3.2", @@ -45,4 +46,4 @@ "devDependencies": { "@theia/cli": "^0.3.2" } -} \ No newline at end of file +} diff --git a/packages/search-in-workspace/compile.tsconfig.json b/packages/search-in-workspace/compile.tsconfig.json new file mode 100644 index 0000000000000..b8b72b49c8822 --- /dev/null +++ b/packages/search-in-workspace/compile.tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../configs/base.tsconfig", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ] +} \ No newline at end of file diff --git a/packages/search-in-workspace/package.json b/packages/search-in-workspace/package.json new file mode 100644 index 0000000000000..29a2e4da161f4 --- /dev/null +++ b/packages/search-in-workspace/package.json @@ -0,0 +1,46 @@ +{ + "name": "@theia/search-in-workspace", + "version": "0.3.2", + "description": "Theia - Search in workspace", + "dependencies": { + "@theia/core": "^0.3.2", + "@theia/editor": "^0.3.2", + "vscode-ripgrep": "^0.7.1-patch.0" + }, + "publishConfig": { + "access": "public" + }, + "theiaExtensions": [ + { + "frontend": "lib/browser/search-in-workspace-frontend-module", + "backend": "lib/node/search-in-workspace-backend-module" + } + ], + "keywords": [ + "theia-extension" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/theia-ide/theia.git" + }, + "bugs": { + "url": "https://github.com/theia-ide/theia/issues" + }, + "homepage": "https://github.com/theia-ide/theia", + "files": [ + "lib", + "src" + ], + "scripts": { + "prepare": "yarn run clean && yarn run build", + "clean": "theiaext clean", + "build": "theiaext build", + "watch": "theiaext watch", + "test": "theiaext test", + "docs": "theiaext docs" + }, + "devDependencies": { + "@theia/ext-scripts": "^0.2.0" + } +} diff --git a/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts b/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts new file mode 100644 index 0000000000000..f38daa08e15e6 --- /dev/null +++ b/packages/search-in-workspace/src/browser/quick-search-in-workspace.ts @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import URI from '@theia/core/lib/common/uri'; +import { QuickOpenService, QuickOpenModel, QuickOpenItem, QuickOpenItemOptions } from '@theia/core/lib/browser/quick-open/'; +import { injectable, inject } from 'inversify'; +import { MenuModelRegistry, MenuContribution, CommandContribution, CommandRegistry, KeybindingContribution, KeybindingRegistry, ILogger } from '@theia/core'; +import { CommonMenus, QuickOpenMode, OpenerService, open, Highlight, QuickOpenOptions } from '@theia/core/lib/browser'; +import { SearchInWorkspaceService } from './search-in-workspace-service'; +import { SearchInWorkspaceResult, SearchInWorkspaceOptions } from '../common/search-in-workspace-interface'; +import { Range } from '@theia/editor/lib/browser'; +import { LabelProvider } from '@theia/core/lib/browser/label-provider'; + +@injectable() +export class QuickSearchInWorkspace implements QuickOpenModel { + private currentSearchId: number = -1; + protected MAX_RESULTS = 100; + + constructor( + @inject(QuickOpenService) protected readonly quickOpenService: QuickOpenService, + @inject(SearchInWorkspaceService) protected readonly searchInWorkspaceService: SearchInWorkspaceService, + @inject(OpenerService) protected readonly openerService: OpenerService, + @inject(LabelProvider) protected readonly labelProvider: LabelProvider, + @inject(ILogger) protected readonly logger: ILogger, + ) { } + + onType(lookFor: string, acceptor: (items: QuickOpenItem[]) => void): void { + // If we have a search pending, it's not relevant anymore, cancel it. + this.cancelCurrentSeach(); + + if (lookFor.length === 0) { + // The user has emptied the search box, call acceptor to + // remove any previously shown results. + acceptor([]); + return; + } + + // Options passed to the search service. + const opts: SearchInWorkspaceOptions = { + maxResults: this.MAX_RESULTS, + }; + + // The array in which we'll keep accumulating search results. + const items: QuickSearchInWorkspaceResultItem[] = []; + + this.searchInWorkspaceService.search(lookFor, { + + onResult: (searchId: number, result: SearchInWorkspaceResult) => { + // Is this result from a previous search? + if (searchId !== this.currentSearchId) { + return; + } + + items.push(new QuickSearchInWorkspaceResultItem(result, this.openerService, this.labelProvider)); + }, + + onDone: (searchId: number, error?: string) => { + if (searchId !== this.currentSearchId) { + this.logger.debug('Search ' + this.currentSearchId + ' has completed, but it\'s not the current search.'); + return; + } + this.logger.debug('Search ' + this.currentSearchId + ' has completed and is the current search.'); + this.currentSearchId = -1; + + if (error) { + this.showFakeResult(error, acceptor); + } else if (items.length !== 0) { + items.sort((a, b) => SearchInWorkspaceResult.compare(a.getResult(), b.getResult())); + acceptor(items); + } else { + this.showFakeResult('No matches :(', acceptor); + } + + }, + }, opts).then(searchId => { + this.currentSearchId = searchId; + }); + } + + showFakeResult(label: string, acceptor: (items: QuickOpenItem[]) => void) { + acceptor([ + new QuickOpenItem({ + label: label, + }), + ]); + } + + // If we have an ongoing search, cancel it. + cancelCurrentSeach() { + if (this.currentSearchId >= 0) { + this.logger.debug('Cancelling search ' + this.currentSearchId); + this.searchInWorkspaceService.cancel(this.currentSearchId); + this.currentSearchId = -1; + } + } + + // Open the quick search in workspace popup. + open() { + const opts: QuickOpenOptions = { + onClose: cancelled => this.cancelCurrentSeach(), + placeholder: 'Search in workspace by regular expression...', + }; + this.quickOpenService.open(this, opts); + } +} + +class QuickSearchInWorkspaceResultItem extends QuickOpenItem { + + private result: SearchInWorkspaceResult; + private openerService: OpenerService; + + constructor(result: SearchInWorkspaceResult, openerService: OpenerService, labelProvider: LabelProvider) { + const resultHl: Highlight = { + start: result.character - 1, + end: result.character + result.length - 1, + }; + + // Show the path relative to the workspace. + const uri = new URI('file://' + result.file); + const file = labelProvider.getName(uri); + const dir = labelProvider.getLongName(uri.parent) + '/'; + + const filenameHl: Highlight = { + start: 0, + end: file.length, + }; + + const opts: QuickOpenItemOptions = { + detail: result.lineText, + detailHighlights: [resultHl], + label: `${file}:${result.line} - ${dir}`, + labelHighlights: [filenameHl], + }; + super(opts); + + this.result = result; + this.openerService = openerService; + } + + run(mode: QuickOpenMode): boolean { + if (mode !== QuickOpenMode.OPEN) { + return false; + } + + // Search results are 1-based, positions in editors are 0-based. + const line = this.result.line - 1; + const character = this.result.character - 1; + const uri = new URI('file://' + this.result.file); + const r = Range.create(line, character, line, character + this.result.length); + open(this.openerService, uri, { selection: r }); + + return true; + } + + getResult(): SearchInWorkspaceResult { + return this.result; + } +} + +const OpenQuickSearchInWorkspaceCommand = { + id: 'QuickSearchInWorkspace.open', + label: "Search in workspace..." +}; + +@injectable() +export class SearchInWorkspaceContributions implements CommandContribution, MenuContribution, KeybindingContribution { + constructor( + @inject(QuickSearchInWorkspace) protected readonly quickSeachInWorkspace: QuickSearchInWorkspace, + ) { } + + registerCommands(registry: CommandRegistry): void { + registry.registerCommand(OpenQuickSearchInWorkspaceCommand, { + execute: what => this.quickSeachInWorkspace.open() + }); + } + + registerMenus(menus: MenuModelRegistry): void { + menus.registerMenuAction(CommonMenus.EDIT_FIND, { + commandId: OpenQuickSearchInWorkspaceCommand.id, + label: OpenQuickSearchInWorkspaceCommand.label, + }); + } + + registerKeybindings(keybindings: KeybindingRegistry): void { + keybindings.registerKeybinding({ + command: OpenQuickSearchInWorkspaceCommand.id, + keybinding: 'ctrlcmd+shift+f', + }); + } +} diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts new file mode 100644 index 0000000000000..3113508928666 --- /dev/null +++ b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-module.ts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule } from "inversify"; +import { SearchInWorkspaceService, SearchInWorkspaceClientImpl } from './search-in-workspace-service'; +import { SearchInWorkspaceServer } from '../common/search-in-workspace-interface'; +import { WebSocketConnectionProvider } from '@theia/core/lib/browser'; +import { QuickSearchInWorkspace, SearchInWorkspaceContributions } from './quick-search-in-workspace'; +import { CommandContribution, MenuContribution, KeybindingContribution } from "@theia/core"; + +export default new ContainerModule(bind => { + bind(QuickSearchInWorkspace).toSelf().inSingletonScope(); + + bind(CommandContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + bind(MenuContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + bind(KeybindingContribution).to(SearchInWorkspaceContributions).inSingletonScope(); + + // The object that gets notified of search results. + bind(SearchInWorkspaceClientImpl).toSelf().inSingletonScope(); + + bind(SearchInWorkspaceService).toSelf().inSingletonScope(); + + // The object to call methods on the backend. + bind(SearchInWorkspaceServer).toDynamicValue(ctx => { + const client = ctx.container.get(SearchInWorkspaceClientImpl); + return WebSocketConnectionProvider.createProxy(ctx.container, '/search-in-workspace', client); + }).inSingletonScope(); +}); diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-service.ts b/packages/search-in-workspace/src/browser/search-in-workspace-service.ts new file mode 100644 index 0000000000000..c737cae744bf4 --- /dev/null +++ b/packages/search-in-workspace/src/browser/search-in-workspace-service.ts @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { injectable, inject } from "inversify"; +import { SearchInWorkspaceServer, SearchInWorkspaceClient, SearchInWorkspaceResult, SearchInWorkspaceOptions } from "../common/search-in-workspace-interface"; +import { WorkspaceService } from "@theia/workspace/lib/browser"; +import URI from "@theia/core/lib/common/uri"; +import { ILogger } from "@theia/core"; + +/** + * Class that will receive the search results from the server. This is separate + * from the SearchInWorkspaceService class only to avoid a cycle in the + * dependency injection. + */ + +@injectable() +export class SearchInWorkspaceClientImpl implements SearchInWorkspaceClient { + private service: SearchInWorkspaceClient; + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + this.service.onResult(searchId, result); + } + onDone(searchId: number, error?: string): void { + this.service.onDone(searchId, error); + } + + setService(service: SearchInWorkspaceClient) { + this.service = service; + } +} + +export type SearchInWorkspaceCallbacks = SearchInWorkspaceClient; + +/** + * Service to search text in the workspace files. + */ + +@injectable() +export class SearchInWorkspaceService implements SearchInWorkspaceClient { + + // All the searches that we have started, that are not done yet (onDone + // with that searchId has not been called). + private pendingSearches = new Map(); + + // Due to the asynchronicity of the node backend, it's possible that we + // start a search, receive an event for that search, and then receive + // the search id for that search.We therefore need to keep those + // events until we get the search id and return it to the caller. + // Otherwise the caller would discard the event because it doesn't know + // the search id yet. + private pendingOnDones: Map = new Map(); + + private lastKnownSearchId: number = -1; + + constructor( + @inject(SearchInWorkspaceServer) protected readonly searchServer: SearchInWorkspaceServer, + @inject(SearchInWorkspaceClientImpl) protected readonly client: SearchInWorkspaceClientImpl, + @inject(WorkspaceService) protected readonly workspaceService: WorkspaceService, + @inject(ILogger) protected readonly logger: ILogger, + ) { + client.setService(this); + } + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + const callbacks = this.pendingSearches.get(searchId); + + if (callbacks) { + callbacks.onResult(searchId, result); + } + } + + onDone(searchId: number, error?: string): void { + const callbacks = this.pendingSearches.get(searchId); + + if (callbacks) { + this.pendingSearches.delete(searchId); + callbacks.onDone(searchId, error); + } else { + if (searchId > this.lastKnownSearchId) { + this.logger.debug(`Got an onDone for a searchId we don't know about (${searchId}), stashing it for later with error = `, error); + this.pendingOnDones.set(searchId, error); + } else { + // It's possible to receive an onDone for a search we have cancelled. Just ignore it. + this.logger.debug(`Got an onDone for a searchId we don't know about (${searchId}), but it's probably an old one, error = `, error); + } + } + } + + // Start a search of the string "what" in the workspace. + async search(what: string, callbacks: SearchInWorkspaceCallbacks, opts?: SearchInWorkspaceOptions): Promise { + const root = await this.workspaceService.root; + + if (!root) { + throw new Error("Search failed: no workspace root."); + } + + const rootUri = new URI(root.uri); + const searchId = await this.searchServer.search(what, rootUri.path.toString(), opts); + this.pendingSearches.set(searchId, callbacks); + this.lastKnownSearchId = searchId; + + this.logger.debug('Service launched search ' + searchId); + + // Check if we received an onDone before search() returned. + if (this.pendingOnDones.has(searchId)) { + this.logger.debug('Ohh, we have a stashed onDone for that searchId'); + const error = this.pendingOnDones.get(searchId); + this.pendingOnDones.delete(searchId); + + // Call the client's searchId, but first give it a + // chance to record the returned searchId. + setTimeout(() => { + this.onDone(searchId, error); + }, 0); + } + + return searchId; + } + + // Cancel an ongoing search. + cancel(searchId: number) { + this.pendingSearches.delete(searchId); + this.searchServer.cancel(searchId); + } +} diff --git a/packages/search-in-workspace/src/common/search-in-workspace-interface.ts b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts new file mode 100644 index 0000000000000..867737a3c85a8 --- /dev/null +++ b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { JsonRpcServer } from "@theia/core"; + +export interface SearchInWorkspaceOptions { + /** + * Maximum number of results to return. Defaults to unlimited. + */ + maxResults?: number; +} + +export interface SearchInWorkspaceResult { + /** + * The path to the file containing the result. + */ + file: string; + + /** + * The (1-based) line number of the result. + */ + line: number; + + /** + * The (1-based) character number in the result line. For UTF-8 files, + * one multi-byte character counts as one character. + */ + character: number; + + /** + * The length of the match, in characters. For UTF-8 files, one + * multi-byte character counts as one character. + */ + length: number; + + /** + * The text of the line containing the result. + */ + lineText: string; +} + +export namespace SearchInWorkspaceResult { + /** + * Sort search in workspace results according to file, line, character position + * and then length. + */ + export function compare(a: SearchInWorkspaceResult, b: SearchInWorkspaceResult) { + if (a.file !== b.file) { + return a.file < b.file ? -1 : 1; + } + + if (a.line !== b.line) { + return a.line - b.line; + } + + if (a.character !== b.character) { + return a.character - b.character; + } + + return a.length - b.length; + } +} + +export const SearchInWorkspaceClient = Symbol("SearchInWorkspaceClient"); +export interface SearchInWorkspaceClient { + /** + * Called by the server for every search match. + */ + onResult(searchId: number, result: SearchInWorkspaceResult): void; + + /** + * Called when no more search matches will come. + */ + onDone(searchId: number, error?: string): void; +} + +export const SearchInWorkspaceServer = Symbol("SearchInWorkspaceServer"); +export interface SearchInWorkspaceServer extends JsonRpcServer { + /** + * Start a search for WHAT in directory ROOT. Return a unique search id. + */ + search(what: string, root: string, opts?: SearchInWorkspaceOptions): Promise; + + /** + * Cancel an ongoing search. + */ + cancel(searchId: number): Promise; + + dispose(): void; +} diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts new file mode 100644 index 0000000000000..8b15393b7ec6e --- /dev/null +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.spec.ts @@ -0,0 +1,410 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import * as chai from 'chai'; +const expect = chai.expect; +import * as temp from 'temp'; +import * as fs from 'fs'; +import { RipgrepSearchInWorkspaceServer } from './ripgrep-search-in-workspace-server'; +import { SearchInWorkspaceClient, SearchInWorkspaceResult } from '../common/search-in-workspace-interface'; +import { Container, ContainerModule } from 'inversify'; +import { ILogger, isWindows } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { RawProcessFactory, RawProcessOptions, RawProcess, ProcessManager } from '@theia/process/lib/node'; +import * as path from 'path'; + +// Allow creating temporary files, but remove them when we are done. +const track = temp.track(); + +// The root dir we'll use to test searching. +let rootDir: string; + +// Remember the content of the test files we create, to validate that the +// reported line text is right. +const fileLines: Map = new Map(); + +// The class under test. +let ripgrepServer: RipgrepSearchInWorkspaceServer; + +// Mock client that accumulates the returned results in a list. +class ResultAccumulator implements SearchInWorkspaceClient { + results: SearchInWorkspaceResult[] = []; + onDoneCallback: () => void; + + constructor(onDoneCallback: () => void) { + this.onDoneCallback = onDoneCallback; + } + + onResult(searchId: number, result: SearchInWorkspaceResult): void { + this.results.push(result); + } + + onDone(searchId: number): void { + // Sort the results, so that the order is predictable. + this.results.sort(SearchInWorkspaceResult.compare); + this.onDoneCallback(); + } +} + +// Create a test file relative to rootDir. +function createTestFile(filename: string, text: string) { + fs.writeFileSync(rootDir + '/' + filename, text); + fileLines.set(filename, text.split('\n')); +} + +before(() => { + rootDir = track.mkdirSync(); + + createTestFile('carrots', `\ +This is a carrot. +Most carrots are orange, but some carrots are not. +Once capitalized, the word carrot looks like this: CARROT. +Carrot is a funny word. +`); + createTestFile('potatoes', `\ +Potatoes, unlike carrots, are generally not orange. But sweet potatoes are, +it's very confusing. +`); + + createTestFile('regexes', `\ +aaa hello x h3lo y hell0h3lllo +hello1 +`); + + fs.mkdirSync(rootDir + '/small'); + createTestFile('small/small', 'A small file.\n'); + + if (!isWindows) { + createTestFile('file:with:some:colons', `\ +Are you looking for this: --foobar? +`); + } + + createTestFile('file with spaces', `\ +Are you looking for this: --foobar? +`); + + createTestFile('utf8-file', `\ +Var är jag? Varför är jag här? +`); + + let lotsOfMatchesText = ''; + for (let i = 0; i < 100000; i++) { + lotsOfMatchesText += 'lots-of-matches\n'; + } + createTestFile('lots-of-matches', lotsOfMatchesText); +}); + +beforeEach(() => { + const container = new Container(); + + const module = new ContainerModule(bind => { + bind(ILogger).to(MockLogger); + bind(RipgrepSearchInWorkspaceServer).toSelf(); + bind(ProcessManager).toSelf().inSingletonScope(); + bind(RawProcess).toSelf().inTransientScope(); + bind(RawProcessFactory).toFactory(ctx => + (options: RawProcessOptions) => { + const child = new Container({ defaultScope: 'Singleton' }); + child.parent = ctx.container; + + child.bind(RawProcessOptions).toConstantValue(options); + return child.get(RawProcess); + } + ); + }); + + container.load(module); + + ripgrepServer = container.get(RipgrepSearchInWorkspaceServer); +}); + +after(() => { + track.cleanupSync(); +}); + +// Compare expected and actual search results. +// +// For convenience, the expected entries do not have their lineText field set +// by individual tests. Using on the file and line fields, this function +// retrieves the expected line text based on what we have written to the test +// files. +// +// The expected entries should also have the file field set relatively to +// rootDir. This function will update the field to contain the absolute path. + +function compareSearchResults(expected: SearchInWorkspaceResult[], actual: SearchInWorkspaceResult[]) { + expect(actual.length).eq(expected.length); + + if (actual.length !== expected.length) { + return; + } + + for (let i = 0; i < actual.length; i++) { + const a = actual[i]; + const e = expected[i]; + + const lines = fileLines.get(e.file); + if (lines) { + const line = lines[e.line - 1]; + e.lineText = line; + e.file = path.join(rootDir, e.file); + + expect(a).deep.eq(e); + } else { + // We don't know this file... + expect.fail(); + } + } +} + +describe('ripgrep-search-in-workspace-server', () => { + // Try some simple patterns with different case. + it('returns 7 results when searching for "carrot"', function (done) { + const pattern = 'carrot'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 2, character: 35, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 3, character: 28, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 3, character: 52, length: pattern.length, lineText: '' }, + { file: 'carrots', line: 4, character: 1, length: pattern.length, lineText: '' }, + { file: 'potatoes', line: 1, character: 18, length: pattern.length, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + it('returns 1 result when searching for "Carrot"', function (done) { + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 4, character: 1, length: 6, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search('Carrot', rootDir); + }); + + it('returns 0 result when searching for "CarroT"', function (done) { + const pattern = 'CarroT'; + + const client = new ResultAccumulator(() => { + compareSearchResults([], client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try something that we know isn't there. + it('finds 0 result when searching for "PINEAPPLE"', function (done) { + const pattern = 'PINEAPPLE'; + + const client = new ResultAccumulator(() => { + compareSearchResults([], client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try a pattern with a space. + it('finds 1 result when searching for "carrots are orange"', function (done) { + const pattern = 'carrots are orange'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with an output size that exceeds the default node buffer size + // (200 * 1024) when spawning a new process. + it('works with a lot of results', function (done) { + // This can take a bit of time. + this.timeout(150000); + const pattern = 'lots-of-matches'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = []; + + for (let i = 1; i <= 100000; i++) { + expected.push({ + file: 'lots-of-matches', + line: i, + character: 1, + length: pattern.length, + lineText: '', + }); + } + + compareSearchResults(expected, client.results); + done(); + }); + + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try limiting the number of returned results. + it('limits the number of returned results', function (done) { + const pattern = 'lots-of-matches'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = []; + + for (let i = 1; i <= 1000; i++) { + expected.push({ + file: 'lots-of-matches', + line: i, + character: 1, + length: pattern.length, + lineText: '', + }); + } + + compareSearchResults(expected, client.results); + done(); + }); + + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir, { + maxResults: 1000, + }); + }); + + // Try with regexes. + it('searches for regexes', function (done) { + const pattern = 'h[e3]l+[o0]'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'regexes', line: 1, character: 5, length: 5, lineText: '' }, + { file: 'regexes', line: 1, character: 13, length: 4, lineText: '' }, + { file: 'regexes', line: 1, character: 20, length: 5, lineText: '' }, + { file: 'regexes', line: 1, character: 25, length: 6, lineText: '' }, + { file: 'regexes', line: 2, character: 1, length: 5, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with a pattern starting with -, and in filenames containing colons and spaces. + it('searches a pattern starting with -', function (done) { + const pattern = '-fo+bar'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'file with spaces', line: 1, character: 28, length: 7, lineText: '' }, + ]; + + if (!isWindows) { + expected.push( + { file: 'file:with:some:colons', line: 1, character: 28, length: 7, lineText: '' } + ); + } + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try with a pattern starting with --, and in filenames containing colons and spaces. + it('searches a pattern starting with --', function (done) { + const pattern = '--fo+bar'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'file with spaces', line: 1, character: 27, length: 8, lineText: '' }, + ]; + + if (!isWindows) { + expected.push( + { file: 'file:with:some:colons', line: 1, character: 27, length: 8, lineText: '' } + ); + } + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try searching in an UTF-8 file. + it('searches in a UTF-8 file', function (done) { + const pattern = ' jag'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'utf8-file', line: 1, character: 7, length: 4, lineText: '' }, + { file: 'utf8-file', line: 1, character: 23, length: 4, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // Try searching a pattern that contains unicode characters. + it('searches a UTF-8 pattern', function (done) { + const pattern = ' h?är'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + { file: 'utf8-file', line: 1, character: 4, length: 3, lineText: '' }, + { file: 'utf8-file', line: 1, character: 20, length: 3, lineText: '' }, + { file: 'utf8-file', line: 1, character: 27, length: 4, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir); + }); + + // A regex that may match an empty string should not return zero-length + // results. Run the test in a directory without big files, because it + // makes rg print all searched lines, which can take a lot of time. + it('doesn\'t return zero-length matches', function (done) { + const pattern = '(hello)?'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceResult[] = [ + ]; + + compareSearchResults(expected, client.results); + done(); + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, rootDir + '/small'); + }); +}); diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts new file mode 100644 index 0000000000000..cc321c2f311b3 --- /dev/null +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { SearchInWorkspaceServer, SearchInWorkspaceOptions, SearchInWorkspaceResult, SearchInWorkspaceClient } from "../common/search-in-workspace-interface"; +import { ILogger } from "@theia/core"; +import { inject, injectable } from "inversify"; +import { RawProcess, RawProcessFactory, RawProcessOptions } from '@theia/process/lib/node'; + +import * as rg from 'vscode-ripgrep'; + +@injectable() +export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { + + // List of ongoing searches, maps search id to a the started rg process. + private ongoingSearches: Map = new Map(); + + // Each incoming search is given a unique id, returned to the client. This is the next id we will assigned. + private nextSearchId: number = 0; + + private client: SearchInWorkspaceClient | undefined; + + // Highlighted red + private readonly FILENAME_START = '^\x1b\\[m\x1b\\[31m'; + private readonly FILENAME_END = '\x1b\\[m:'; + // Highlighted green + private readonly LINE_START = '^\x1b\\[m\x1b\\[32m'; + private readonly LINE_END = '\x1b\\[m:'; + // Highlighted yellow + private readonly CHARACTER_START = '^\x1b\\[m\x1b\\[33m'; + private readonly CHARACTER_END = '\x1b\\[m:'; + // Highlighted blue + private readonly MATCH_START = '\x1b\\[m\x1b\\[34m\x1b\\[1m'; + private readonly MATCH_END = '\x1b\\[m'; + + constructor( + @inject(ILogger) protected readonly logger: ILogger, + @inject(RawProcessFactory) protected readonly rawProcessFactory: RawProcessFactory, + ) { } + + setClient(client: SearchInWorkspaceClient | undefined): void { + this.client = client; + } + + // Search for the string WHAT in directory ROOT. Return the assigned search id. + search(what: string, root: string, opts?: SearchInWorkspaceOptions): Promise { + // Start the rg process. Use --vimgrep to get one result per + // line, --color=always to get color control characters that + // we'll use to parse the lines. + const searchId = this.nextSearchId++; + const processOptions: RawProcessOptions = { + command: rg.rgPath, + args: ["--vimgrep", "-S", "--color=always", + "--colors=path:fg:red", + "--colors=line:fg:green", + "--colors=column:fg:yellow", + "--colors=match:fg:blue", + "-e", what, root], + }; + const process: RawProcess = this.rawProcessFactory(processOptions); + this.ongoingSearches.set(searchId, process); + + process.onError(error => { + // tslint:disable-next-line:no-any + let errorCode = (error as any).code; + + // Try to provide somewhat clearer error messages, if possible. + if (errorCode === 'ENOENT') { + errorCode = 'could not find the ripgrep (rg) binary'; + } else if (errorCode === 'EACCES') { + errorCode = 'could not execute the ripgrep (rg) binary'; + } + + const errorStr = `An error happened while searching (${errorCode}).`; + this.wrapUpSearch(searchId, errorStr); + }); + + // Running counter of results. + let numResults = 0; + + // Buffer to accumulate incoming output. + let databuf: string = ""; + + const lastMatch = { + file: '', + line: 0, + index: 0, + }; + + process.output.on('data', (chunk: string) => { + // We might have already reached the max number of + // results, sent a TERM signal to rg, but we still get + // the data that was already output in the mean time. + // It's not necessary to return early here (the check + // for maxResults below would avoid sending extra + // results), but it avoids doing unnecessary work. + if (opts && opts.maxResults && numResults >= opts.maxResults) { + return; + } + + databuf += chunk; + + while (1) { + // Check if we have a complete line. + const eolIdx = databuf.indexOf('\n'); + if (eolIdx < 0) { + break; + } + + // Get and remove the line from the data buffer. + let lineBuf = databuf.slice(0, eolIdx); + databuf = databuf.slice(eolIdx + 1); + + // Extract the various fields using the ANSI + // control characters for colors as guides. + + // Extract filename (magenta). + const filenameRE = new RegExp(`${this.FILENAME_START}(.+?)${this.FILENAME_END}`); + let match = filenameRE.exec(lineBuf); + if (!match) { + continue; + } + + const filename = match[1]; + lineBuf = lineBuf.slice(match[0].length); + + // Extract line number (green). + const lineRE = new RegExp(`${this.LINE_START}(\\d+)${this.LINE_END}`); + match = lineRE.exec(lineBuf); + if (!match) { + continue; + } + + const line = +match[1]; + lineBuf = lineBuf.slice(match[0].length); + + // Extract character number (column), but don't + // do anything with it. ripgrep reports the + // offset in bytes, which is not good when + // dealing with multi-byte UTF-8 characters. + const characterNumRE = new RegExp(`${this.CHARACTER_START}(\\d+)${this.CHARACTER_END}`); + match = characterNumRE.exec(lineBuf); + if (!match) { + continue; + } + + lineBuf = lineBuf.slice(match[0].length); + + // If there are two matches in a line, + // --vimgrep will make rg output two lines, but + // both matches will be highlighted in both + // lines. If we have consecutive matches at + // the same file / line, make sure to pick the + // right highlighted match. + if (lastMatch.file === filename && lastMatch.line === line) { + lastMatch.index++; + } else { + lastMatch.file = filename; + lastMatch.line = line; + lastMatch.index = 0; + } + + // Extract the match text (red). + const matchRE = new RegExp(`${this.MATCH_START}(.*?)${this.MATCH_END}`); + + let characterNum = 0; + + let matchWeAreLookingFor: RegExpMatchArray | undefined = undefined; + for (let i = 0; ; i++) { + const nextMatch = lineBuf.match(matchRE); + + if (!nextMatch) { + break; + } + + // Just to make typescript happy. + if (nextMatch.index === undefined) { + break; + } + + if (i === lastMatch.index) { + matchWeAreLookingFor = nextMatch; + characterNum = nextMatch.index + 1; + } + + // Remove the control characters around the match. This allows to: + + // - prepare the line text so it can be returned to the client without control characters + // - get the character index of subsequent matches right + + lineBuf = + lineBuf.slice(0, nextMatch.index) + + nextMatch[1] + + lineBuf.slice(nextMatch.index + nextMatch[0].length); + } + + if (!matchWeAreLookingFor || characterNum === 0) { + continue; + } + + if (matchWeAreLookingFor[1].length === 0) { + continue; + } + + const result: SearchInWorkspaceResult = { + file: filename, + line: line, + character: characterNum, + length: matchWeAreLookingFor[1].length, + lineText: lineBuf, + }; + + numResults++; + if (this.client) { + this.client.onResult(searchId, result); + } + + // Did we reach the maximum number of results? + if (opts && opts.maxResults && numResults >= opts.maxResults) { + process.kill(); + this.wrapUpSearch(searchId); + break; + } + } + }); + + process.output.on('end', () => { + // If we reached maxResults, we should have already + // wrapped up the search. Returning early avoids + // logging a warning message in wrapUpSearch. + if (opts && opts.maxResults && numResults >= opts.maxResults) { + return; + } + + this.wrapUpSearch(searchId); + }); + + return Promise.resolve(searchId); + } + + // Cancel an ongoing search. Trying to cancel a search that doesn't exist isn't an + // error, otherwise we'd have to deal with race conditions, where a client cancels a + // search that finishes normally at the same time. + cancel(searchId: number): Promise { + const process = this.ongoingSearches.get(searchId); + if (process) { + process.kill(); + this.wrapUpSearch(searchId); + } + + return Promise.resolve(); + } + + // Send onDone to the client and clean up what we know about search searchId. + private wrapUpSearch(searchId: number, error?: string) { + if (this.ongoingSearches.delete(searchId)) { + if (this.client) { + this.logger.debug("Sending onDone for " + searchId, error); + this.client.onDone(searchId, error); + } else { + this.logger.debug("Wrapping up search " + searchId + " but no client"); + } + } else { + this.logger.debug("Trying to wrap up a search we don't know about " + searchId); + } + } + + dispose(): void { + } +} diff --git a/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts b/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts new file mode 100644 index 0000000000000..a3bf31ca74f73 --- /dev/null +++ b/packages/search-in-workspace/src/node/search-in-workspace-backend-module.ts @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2017-2018 Erisson and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +import { ContainerModule } from "inversify"; +import { ConnectionHandler, JsonRpcConnectionHandler } from "@theia/core/lib/common"; +import { SearchInWorkspaceServer, SearchInWorkspaceClient } from "../common/search-in-workspace-interface"; +import { RipgrepSearchInWorkspaceServer } from "./ripgrep-search-in-workspace-server"; + +export default new ContainerModule(bind => { + bind(SearchInWorkspaceServer).to(RipgrepSearchInWorkspaceServer); + bind(ConnectionHandler).toDynamicValue(ctx => + new JsonRpcConnectionHandler + ('/search-in-workspace', client => { + const server = ctx.container.get(SearchInWorkspaceServer); + server.setClient(client); + client.onDidCloseConnection(() => server.dispose()); + return server; + }) + ); +}); diff --git a/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts b/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts new file mode 100644 index 0000000000000..a48e154d34a3e --- /dev/null +++ b/packages/search-in-workspace/src/node/typings/vscode-ripgrep.d.ts @@ -0,0 +1,3 @@ +declare module 'vscode-ripgrep' { + export const rgPath: string; +}