Consolidate code intel extensions

.eslintrc.js

@@ -0,0 +1,12 @@
+module.exports = {
+  extends: '@sourcegraph/eslint-config',
+  env: {
+    browser: true,
+    node: true,
+  },
+  parserOptions: {
+    project: 'tsconfig.json',
+  },
+  ignorePatterns: ['temp'],
+  rules: {},
+}

.github/CODEOWNERS

@@ -1,9 +1,14 @@
 # Lines starting with '#' are comments.
 # Each line is a file pattern followed by one or more owners.
 # https://help.github.com/articles/about-codeowners/
+#
+# Order is important; the last matching pattern takes the most
+# precedence.
 
 * @sourcegraph/code-intel
 
-# Configured through Renovate
-package.json
-yarn.lock
+# These are configured through Renovate config.
+# See ../renovate.json and https://github.com/sourcegraph/renovate-config/blob/master/renovate.json
+# This is so that automerged PRs do not trigger email notification spam.
+**/package.json
+**/yarn.lock

.gitignore

@@ -1,8 +1,7 @@
-.cache/
+temp
+dist
+node_modules
+.cache
 .nyc_output/
-coverage/
-dist/
-node_modules/
-temp/
-lib/
+.eslintcache
 yarn-error.log

.mocharc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  require: ['ts-node/register'],
+  timeout: '10s',
+  slow: '2s',
+}

.prettierignore

@@ -1,8 +1,6 @@
-.cache/
-.nyc_output/
-coverage/
-dist/
-node_modules/
-temp/
-lib/
-yarn-error.log
+temp
+dist
+node_modules
+.cache
+samples
+.nyc_output

.vscode/settings.json

@@ -1,8 +1,8 @@
 {
   "files.exclude": {
-    ".cache/": true,
-    "dist/": true,
-    "lib/": true
+    "**/.cache/": true,
+    "**/dist/": true,
+    "**/node_modules/": true
   },
   "typescript.tsdk": "node_modules/typescript/lib"
 }

README.md

@@ -1,65 +1,45 @@
-# Basic code intelligence for Sourcegraph
+# Sourcegraph code intelligence extensions
 
-[![Build Status](https://travis-ci.org/sourcegraph/sourcegraph-basic-code-intel.svg?branch=master)](https://travis-ci.org/sourcegraph/sourcegraph-basic-code-intel)
-[![codecov](https://codecov.io/gh/sourcegraph/sourcegraph-basic-code-intel/branch/master/graph/badge.svg)](https://codecov.io/gh/sourcegraph/sourcegraph-basic-code-intel)
+This repository contains the code for the [Sourcegraph extensions that provide code intelligence](https://sourcegraph.com/extensions?query=category%3A%22Programming+languages%22). These extensions provide precise code intelligence via LSIF and Language Servers, and provides fuzzy code intelligence using a combination of ctags and search.
 
-basic-code-intel provides fuzzy code intelligence using a combination of ctags and search. All of the [programming language Sourcegraph extensions](https://sourcegraph.com/extensions?query=category%3A%22Programming+languages%22) use the basic-code-intel npm package that lives in this repository.
+## Repository structure
 
-## Development
-
-First install [goreman](https://github.com/mattn/goreman):
-
-```
-$ go get github.com/mattn/goreman
-```
+-   [`src/extensions/go`](./src/extensions/go) The Go extension (supports LSP)
+-   [`src/extensions/typescript`](./src/extensions/typescript) The TypeScript extension (supports LSP)
+-   [`src/extensions/template`](./src/extensions/template) Template for all other languages (**does not** support LSP)
+-   [`shared/language-specs`](./shared/language-specs) Language definitions, which are fed into the template to instantiate many language extensions
+-   [`dev/scripts/generate.ts`](./scripts/generate.ts) Generates language extensions given language definitions
+-   [`dev/scripts/publish.ts`](./scripts/publish.ts) Publishes language extensions
 
-Then run:
-
-```
-$ ./dev
-```
+## Development
 
-Open up your Sourcegraph settings https://sourcegraph.com/users/you/settings and disable the language extensions you're developing:
+1. Run `yarn`
+1. Run `yarn --cwd extensions/{go,typescript,template} run serve` (pick one, `template` includes all others)
+1. Open up your Sourcegraph settings https://sourcegraph.com/users/you/settings and disable the language extensions you're developing:
 
 ```json
+{
+  ...
   "extensions": {
       "sourcegraph/cpp": false,
       ...
   }
+}
 ```
 
-Then [sideload the extension](https://docs.sourcegraph.com/extensions/authoring/local_development) (http://localhost:1234) on your Sourcegraph instance and refresh the page. Make sure you don't see two of the same language extension.
-
-Open the browser Network tab and you should start seeing `graphql?Search` calls when you hover over tokens.
-
-## Anatomy of this repository
-
-- [package/](./package/) The basic-code-intel npm package https://www.yarnpkg.com/en/package/@sourcegraph/basic-code-intel (exposes basic-code-intel functionality to [Go](https://github.com/sourcegraph/sourcegraph-go) and [TypeScript](https://github.com/sourcegraph/sourcegraph-typescript), which live in other repositories)
-- [template/](./template/) A template language extension that uses that package (for development purposes)
-- [generator/](./generator/) A script that generates all of the various basic language extensions (all but [Go](https://github.com/sourcegraph/sourcegraph-go) and [TypeScript](https://github.com/sourcegraph/sourcegraph-typescript), which live in separate repositories)
-
-## Updating the basic-code-intel package
-
-After you make a change to the basic-code-intel package and bump the version number in [`package/package.json`](./package/package.json), build and publish it to npm:
-
-```
-cd package
-yarn run build
-npm publish
-```
-
-Then, update the version in the downstream language extensions by bumping the version in `package.json`.
-
-Finally, publish the extensions with `bash generator/generate.sh --publish`.
+4. [Sideload the extension](https://docs.sourcegraph.com/extensions/authoring/local_development) (hit OK on the alert to accept the default URL http://localhost:1234) on your Sourcegraph instance and refresh the page. Make sure you don't see two of the same language extension in the **Ext** menu.
 
-## Creating your own language extension based on @sourcegraph/basic-code-intel
+## Publishing extensions
 
-1. Create a new extension https://docs.sourcegraph.com/extensions/authoring/creating
-1. Take a look at the [Go](https://github.com/sourcegraph/sourcegraph-go/blob/master/src/lang-go.ts) extension to see how to use the `@sourcegraph/basic-code-intel` package
-1. Change the file extensions, test, and [publish](https://docs.sourcegraph.com/extensions/authoring/publishing)
+-   For Go: `yarn --cwd extensions/go run publish`
+-   For TypeScript: `yarn --cwd extensions/typescript run publish`
+-   For all others:
+    1. `yarn run generate` (optional: to generate only a few languages, pass `--languages lang1,lang2`)
+    1. `yarn run publish` (supports `--languages`, too)
 
-## Adding a new sourcegraph/sourcegraph-LANG extension
+## Adding a language extension
 
 1. Add an entry to `languages` in [`generator/src/main.ts`](generator/src/main.ts)
 1. (optional, to enable jump to definition) Ensure the language is present in the command line arguments to universal-ctags https://github.com/sourcegraph/sourcegraph/blob/21efc6844838e773b9a8f4a7ba1d5628e8076984/cmd/symbols/internal/pkg/ctags/parser.go#L71
-1. Run `bash generate.sh --languages <language name> --publish`
+1. Make sure there is a mapping entry for the `languageID` in https://github.com/sourcegraph/sourcegraph/blob/master/shared/src/languages.ts#L40
+1. Generate and publish the extension as described in the previous section

buildkite.yml

@@ -2,6 +2,14 @@ env:
   FORCE_COLOR: '1'
 
 steps:
-  - command: |-
-      ./ci
-    label: ':pipeline:'
+  - label: ':white_check_mark:'
+    command: ./dev/ci/check-all.sh
+
+  - label: ':lipstick: :lint-roller:'
+    command: ./dev/ci/lint.sh
+
+  - label: ':jest:'
+    command: ./dev/ci/test.sh
+
+  - label: ':typescript:'
+    command: ./dev/ci/build-all.sh

dev/ci/build-all.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")/../..
+
+# mutex is necessary since CI runs various yarn installs in parallel
+yarn --mutex network --frozen-lockfile --network-timeout 60000
+
+pushd extensions/go
+yarn run build
+popd
+
+pushd extensions/template
+yarn run build
+popd
+
+pushd extensions/typescript
+yarn run build
+popd

dev/ci/check-all.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")
+
+CHECKS=(
+    ./yarn-deduplicate.sh
+)
+
+for c in $CHECKS; do
+    $c
+done

dev/ci/lint.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")
+
+./yarn-run.sh prettier-check # tslint eslint

dev/ci/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")
+
+./yarn-run.sh "test --verbose"

dev/ci/yarn-deduplicate.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")
+
+# Disabled in CI for now as we require explicit resolutions
+# ./yarn-run.sh "yarn-deduplicate --fail --list --strategy fewer ./yarn.lock" || {
+#     echo 'yarn.lock contains duplicate dependencies. Please run `yarn deduplicate` and commit the result.'
+#     exit 1
+# }

dev/ci/yarn-run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -ex
+cd $(dirname "${BASH_SOURCE[0]}")/../..
+
+# mutex is necessary since CI runs various yarn installs in parallel
+yarn --mutex network --frozen-lockfile --network-timeout 60000
+
+for cmd in "$@"; do
+    yarn -s run $cmd
+done

dev/extract-ctags/README.md

@@ -0,0 +1 @@
+TODO

dev/scripts/args.ts

@@ -0,0 +1,34 @@
+import * as yargs from 'yargs'
+import {
+    LanguageSpec,
+    languageSpecs,
+} from '../../shared/language-specs/languages'
+
+const blacklist = ['go', 'typescript']
+
+export function findLanguageSpecs(): LanguageSpec[] {
+    const args = yargs
+        .nargs('languages', 1)
+        .describe('l', 'A list of (comma-separated) languages to generate')
+        .alias('l', 'languages')
+        .strict().argv as { language?: string }
+
+    const candidates = languageSpecs.filter(
+        s => !blacklist.includes(s.handlerArgs.languageID)
+    )
+
+    if (!args.language) {
+        return candidates
+    }
+
+    return args.language.split(',').map(languageID => {
+        const spec = candidates.find(
+            spec => spec.handlerArgs.languageID === languageID
+        )
+        if (!spec) {
+            throw new Error(`Unknown language ${languageID}.`)
+        }
+
+        return spec
+    })
+}