ci: build and push docker images (#42)

This is a port and improvement of our docker images in infrastructure. It will
build and push zoekt-webserver and zoekt-indexserver images on every push to
master. We use GitHub actions to achieve this. It uses the same version format
as we previously did.

The improvement is to share a base image which contains all the zoekt binaries
and dependencies. The different images then just copy out what they need and
do the sourcegraph specific customizations. This helps us share layers in a
cluster as well as reduce the time it takes to build.

.ctags.d/additional-languages.ctags

@@ -0,0 +1,13 @@
+# To avoid the warning:
+#
+#     Unknown ctags kind: foo
+#
+# Make sure that all ctags kinds in this file
+# are handled in ctagsKindToLSPSymbolKind in
+# cmd/frontend/graphqlbackend/search_symbols.go
+
+# After changing this file, run a dev server and check for any other warnings
+# from the symbols service.
+
+# When you want to add a parser, make a file in this directory with
+# .ctags as suffix.

.ctags.d/clojure.ctags

@@ -0,0 +1,11 @@
+--regex-clojure=/def ([A-Za-z0-9_!?+*<>=-]+)/\1/v,variable/
+--regex-clojure=/defmacro ([A-Za-z0-9_!?+*<>=-]+)/\1/m,macro/
+--regex-clojure=/defprotocol ([A-Za-z0-9_!?+*<>=-]+)/\1/p,protocol/
+--regex-clojure=/defrecord ([A-Za-z0-9_!?+*<>=-]+)/\1/r,record/
+--regex-clojure=/deftype ([A-Za-z0-9_!?+*<>=-]+)/\1/t,type/
+--regex-clojure=/s\/def ([A-Za-z0-9_!?+*<>=-]+)/\1/v,variable/
+--regex-clojure=/s\/defn ([A-Za-z0-9_!?+*<>=-]+)/\1/f,function/
+--regex-clojure=/s\/defmacro ([A-Za-z0-9_!?+*<>=-]+)/\1/m,macro/
+--regex-clojure=/s\/defprotocol ([A-Za-z0-9_!?+*<>=-]+)/\1/p,protocol/
+--regex-clojure=/s\/defrecord ([A-Za-z0-9_!?+*<>=-]+)/\1/r,record/
+--regex-clojure=/s\/deftype ([A-Za-z0-9_!?+*<>=-]+)/\1/t,type/

.ctags.d/graphql.ctags

@@ -0,0 +1,11 @@
+--langdef=GraphQL
+--langmap=GraphQL:.graphql
+--regex-graphql=/^[ \t]*([_A-Za-z][_A-Za-z0-9]*)[\(:]/\1/v,field/
+--regex-graphql=/^[ \t]*enum[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/e,enum/
+--regex-graphql=/^[ \t]*fragment[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/f,fragment/
+--regex-graphql=/^[ \t]*input[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/i,input/
+--regex-graphql=/^[ \t]*interface[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/n,interface/
+--regex-graphql=/^[ \t]*mutation[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/m,mutation/
+--regex-graphql=/^[ \t]*query[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/q,query/
+--regex-graphql=/^[ \t]*type[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/t,type/
+--regex-graphql=/^[ \t]*scalar[ \t]+([_A-Za-z][_0-9A-Za-z]*)/\1/t,type/

.ctags.d/groovy.ctags

@@ -0,0 +1,10 @@
+--langdef=Groovy
+--langmap=Groovy:.groovy
+--regex-groovy=/^[ \t]*package[ \t]+([a-zA-Z0-9.-_]+)/\1/p,package/
+--regex-groovy=/^[ \t]*((private|public)[ \t]*)?(abstract|final|static)?[ \t]*class[ \t]+([A-Za-z0-9_]+)/\4/c,class/
+--regex-groovy=/^[ \t]*((private|public)[ \t]*)?interface[ \t]+([A-Za-z0-9_]+)/\3/n,interface/
+--regex-groovy=/^[ \t]*((private|public)[ \t]*)?trait[ \t]+([A-Za-z0-9_]+)/\3/t,trait/
+--regex-groovy=/^[ \t]*((private|public)[ \t]*)?enum[ \t]+([A-Za-z0-9_]+)/\3/e,enum/
+--regex-groovy=/^[ \t]*(public|private|protected[ \t]*)*([a-zA-Z0-9_]{3,})\([A-Za-z0-9 _,]*\)[ \t]+/\2/r,constructor/
+--regex-groovy=/^[ \t]*((abstract|final|public|private|protected|static)[ \t]*)*(def|void|byte|int|short|long|float|double|boolean|char|[A-Z][a-zA-Z0-9_]*)[ \t]+([a-zA-Z0-9_]+)\(/\4/m,method/
+--regex-groovy=/^[ \t]*((final|public|private|protected|static|synchronized)[ \t]*)*(def|byte|int|short|long|float|double|boolean|char|[A-Z][A-Za-z0-9_]*)[ \t]+([a-zA-Z0-9_]+)[ \t]*([\/]+.*)?$/\4/v,field/

.ctags.d/haskell.ctags

@@ -0,0 +1,8 @@
+# See additional-language.ctags for maintaining this file
+--langdef=haskell
+--langmap=haskell:.hs
+--regex-haskell=/^module[ \t]*([A-Z][a-zA-Z0-9'_.]*)/\1/m,module/
+--regex-haskell=/^type[ \t]*([A-Z][a-zA-Z0-9'_]*)./\1/a,alias/
+--regex-haskell=/^class[ \t]*([A-Z][a-zA-Z0-9'_]*)/\1/c,class/
+--regex-haskell=/^(data|newtype)[ \t]*([A-Z][a-zA-Z0-9'_]*)/\2/t,type/
+--regex-haskell=/^([a-z_][a-zA-Z0-9'_]*) ::/\1/v,constant/

.ctags.d/jsonnet.ctags

@@ -0,0 +1,6 @@
+--langdef=Jsonnet
+--langmap=Jsonnet:.jsonnet
+--langmap=Jsonnet:.libsonnet
+--regex-jsonnet=/^[ \t]*local ([A-Za-z0-9_]+)/\1/v,variable/
+--regex-jsonnet=/^[ \t]*([A-Za-z0-9_]+):::?/\1/m,member/
+--regex-jsonnet=/^[ \t]*([A-Za-z0-9_]+)\([^)]*\):::?/\1/f,function/

.ctags.d/kotlin.ctags

@@ -0,0 +1,12 @@
+--langdef=kotlin
+--langmap=kotlin:+.kt
+--langmap=kotlin:+.kts
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy|inner|open)[[:space:]]*)*(private[^ ]*|protected)?[[:space:]]*class[[:space:]]+([[:alnum:]_:]+)/\4/c,class/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*(private[^ ]*|protected)?[[:space:]]*object[[:space:]]+([[:alnum:]_:]+)/\4/o,object/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy|open)[[:space:]]*)*(private[^ ]*|protected)?[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*data class[[:space:]]+([[:alnum:]_:]+)/\6/c,class/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*(private[^ ]*|protected)?[[:space:]]*interface[[:space:]]+([[:alnum:]_:]+)/\4/i,interface/
+--regex-kotlin=/^[[:space:]]*type[[:space:]]+([[:alnum:]_:]+)/\1/T,type/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy|override|open|private[^ ]*(\[[a-z]*\])*|protected)[[:space:]]*)*fun[[:space:]]+([[:alnum:]_:]+)/\4/m,method/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy|override|private[^ ]*|protected)[[:space:]]*)*(const[[:space:]])*val[[:space:]]+([[:alnum:]_]+)/\4/C,constant/
+--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy|override|private[^ ]*|protected)[[:space:]]*)*(lateinit[[:space:]]*)*var[[:space:]]+([[:alnum:]_]+)/\4/v,variable/
+--regex-kotlin=/^[[:space:]]*package[[:space:]]+([[:alnum:]_.:]+)/\1/p,package/

.ctags.d/scala.ctags

@@ -0,0 +1,13 @@
+# See additional-language.ctags for maintaining this file
+--langdef=scala
+--langmap=scala:.scala
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy)[ \t]*)*(private[^ ]*|protected)?[ \t]*class[ \t]+([a-zA-Z0-9_]+)/\4/c,class/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy)[ \t]*)*(private[^ ]*|protected)?[ \t]*object[ \t]+([a-zA-Z0-9_]+)/\4/o,object/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy)[ \t]*)*(private[^ ]*|protected)?[ \t]*((abstract|final|sealed|implicit|lazy)[ \t ]*)*case class[ \t ]+([a-zA-Z0-9_]+)/\6/c,class/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy)[ \t]*)*(private[^ ]*|protected)?[ \t]*case object[ \t]+([a-zA-Z0-9_]+)/\4/o,object/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy)[ \t]*)*(private[^ ]*|protected)?[ \t]*trait[ \t]+([a-zA-Z0-9_]+)/\4/i,interface/
+--regex-scala=/^[ \t]*type[ \t]+([a-zA-Z0-9_]+)/\1/T,type/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy|override|private[^ ]*(\[[a-z]*\])*|protected)[ \t]*)*def[ \t]+([a-zA-Z0-9_]+)/\4/m,method/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy|override|private[^ ]*|protected)[ \t]*)*val[ \t]+([a-zA-Z0-9_]+)/\3/v,variable/
+--regex-scala=/^[ \t]*((abstract|final|sealed|implicit|lazy|override|private[^ ]*|protected)[ \t]*)*var[ \t]+([a-zA-Z0-9_]+)/\3/v,variable/
+--regex-scala=/^[ \t]*package[ \t]+([a-zA-Z0-9_.]+)/\1/p,package/

.ctags.d/swift.ctags

@@ -0,0 +1,11 @@
+--langdef=swift
+--langmap=swift:.swift
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*class[[:space:]]+([[:alnum:]_]+)/\3/c,class/
+--regex-swift=/^[[:space:]]*public[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|final)[[:space:]])*let[[:space:]]+([[:alnum:]_]+)/\3/C,constant/
+--regex-swift=/^[[:space:]]*public[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|final|lazy|mutating|nonmutating|optional|override|required|unowned|weak)[[:space:]])*var[[:space:]]+([[:alnum:]_]+)/\3/v,variable/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*enum[[:space:]]+([[:alnum:]_]+)/\3/e,enum/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var|convenience|dynamic|final|mutating|nonmutating|optional|override|required)[[:space:]])*func[[:space:]]+([[:alnum:]_]+)/\3/f,function/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*protocol[[:space:]]+([[:alnum:]_]+)/\3/i,interface/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*struct[[:space:]]+([[:alnum:]_]+)/\3/s,struct/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*extension[[:space:]]+([[:alnum:]_]+)/\3/d,define/
+--regex-swift=/^[[:space:]]*((associatedtype|class|deinit|enum|extension|fileprivate|func|import|init|inout|internal|let|open|operator|private|protocol|public|static|struct|subscript|typealias|var)[[:space:]])*typealias[[:space:]]+([[:alnum:]_]+)/\3/a,alias/

.ctags.d/thrift.ctags

@@ -0,0 +1,9 @@
+--langdef=Thrift
+--langmap=Thrift:.thrift
+--regex-thrift=/^[ \t]*exception[ \t]+([a-zA-Z0-9_]+)/\1/x,exception/
+--regex-thrift=/^[ \t]*enum[ \t]+([a-zA-Z0-9_]+)/\1/e,enum/
+--regex-thrift=/^[ \t]*struct[ \t]+([a-zA-Z0-9_]+)/\1/s,struct/
+--regex-thrift=/^[ \t]*service[ \t]+([a-zA-Z0-9_]+)/\1/v,service/
+--regex-thrift=/^[ \t]*[0-9]+:[ \t]+([a-zA-Z0-9_]+)[ \t]+([a-zA-Z0-9_]+)/\2/m,member/
+--regex-thrift=/^[ \t]*([a-zA-Z0-9_]+)[ \t]+=/\1/a,value/
+--regex-thrift=/^[ \t]*[a-zA-Z0-9_<>]+[ \t]+([a-zA-Z0-9_]+)[ \t]*\(/\1/f,function/

.ctags.d/tsx.ctags

@@ -0,0 +1,20 @@
+# See additional-language.ctags for maintaining this file
+
+# To avoid the warning:
+#
+#     Don't reuse the kind letter `c' in a language typescript (old: "classes", new: "modules")
+#
+# Make sure there's a 1-1 correspondence between kind letters and kind names.
+
+--langdef=tsx
+--langmap=tsx:.tsx
+--regex-tsx=/^[ \t]*(export[ \t]+([a-z]+[ \t]+)?)?class[ \t]+([a-zA-Z0-9_$]+)/\3/c,class/
+--regex-tsx=/^[ \t]*(declare[ \t]+)?namespace[ \t]+([a-zA-Z0-9_$]+)/\2/n,module/
+--regex-tsx=/^[ \t]*(export[ \t]+)?module[ \t]+([a-zA-Z0-9_$]+)/\2/n,module/
+--regex-tsx=/^[ \t]*(export[ \t]+)?(default[ \t]+)?(async[ \t]+)?function[ \t]+([a-zA-Z0-9_$]+)/\4/f,function/
+--regex-tsx=/^[ \t]*export[ \t]+(var|let|const)[ \t]+([a-zA-Z0-9_$]+)/\2/v,variable/
+--regex-tsx=/^[ \t]*(var|let|const)[ \t]+([a-zA-Z0-9_$]+)[ \t]*=[ \t]*function[ \t]*[*]?[ \t]*\(\)/\2/v,variable/
+--regex-tsx=/^[ \t]*(export[ \t]+)?(public|protected|private)[ \t]+(static[ \t]+)?(abstract[ \t]+)?(((get|set)[ \t]+)|(async[ \t]+[*]*[ \t]*))?([a-zA-Z1-9_$]+)/\9/m,member/
+--regex-tsx=/^[ \t]*(export[ \t]+)?interface[ \t]+([a-zA-Z0-9_$]+)/\2/i,interface/
+--regex-tsx=/^[ \t]*(export[ \t]+)?type[ \t]+([a-zA-Z0-9_$]+)/\2/t,type/
+--regex-tsx=/^[ \t]*(export[ \t]+)?enum[ \t]+([a-zA-Z0-9_$]+)/\2/e,enum/

.dockerignore

@@ -0,0 +1,3 @@
+.git
+Dockerfile*
+.dockerignore

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+on: push
+name: CI
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14.x
+    - name: checkout
+      uses: actions/checkout@v2
+    - name: test
+      run: go test ./...
+
+  # We build a shared docker image called "zoekt". This is not pushed, but is
+  # used for creating the indexserver and webserver images.
+  docker:
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+
+    # This is the same tag we used for zoekt in the infrastructure repo.
+    - name: version
+      id: version
+      run: echo "::set-output name=value::`date -u +'0.0.%Y%m%d%H%M%S'`-`git rev-parse --short HEAD`"
+
+    - name: build-zoekt
+      uses: docker/build-push-action@v1
+      with:
+        repository: zoekt
+        tags: "latest"
+        add_git_labels: "true"
+        push: "false"
+
+    - name: build-push-webserver
+      uses: docker/build-push-action@v1
+      with:
+        repository: sourcegraph/zoekt-webserver
+        tags: ${{ steps.version.outputs.value }},latest
+        dockerfile: Dockerfile.webserver
+        add_git_labels: "true"
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: build-push-indexserver
+      uses: docker/build-push-action@v1
+      with:
+        repository: sourcegraph/zoekt-indexserver
+        tags: ${{ steps.version.outputs.value }},latest
+        dockerfile: Dockerfile.indexserver
+        add_git_labels: "true"
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}

Dockerfile

@@ -0,0 +1,42 @@
+FROM golang:alpine AS builder
+
+RUN apk add --no-cache ca-certificates
+
+ENV CGO_ENABLED=0 GO111MODULE=on
+WORKDIR /go/src/github.com/google/zoekt
+
+# Cache dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . ./
+RUN go install ./cmd/...
+
+FROM alpine AS ctags
+
+RUN apk add --no-cache --virtual build-deps ca-certificates curl jansson-dev \
+    libseccomp-dev linux-headers autoconf pkgconfig make automake \
+    gcc g++ binutils
+
+ENV CTAGS_VERSION=1a94658c2cb47304d0e9811d49e0a5f48bdb3a0a
+
+RUN curl -fsSL -o ctags.tar.gz "https://codeload.github.com/universal-ctags/ctags/tar.gz/$CTAGS_VERSION" && \
+    tar -C /tmp -xzf ctags.tar.gz && cd /tmp/ctags-$CTAGS_VERSION && \
+    ./autogen.sh && LDFLAGS=-static ./configure --program-prefix=universal- --enable-json --enable-seccomp && \
+    make -j8 && make install && cd && \
+    rm -rf /tmp/ctags-$CTAGS_VERSION && \
+    apk --no-cache --purge del build-deps
+
+FROM alpine AS zoekt
+
+RUN apk add --no-cache git ca-certificates bind-tools tini
+
+COPY .ctags.d /.ctags.d
+COPY --from=ctags /usr/local/bin/universal-* /usr/local/bin/
+COPY --from=builder /go/bin/* /usr/local/bin/
+
+# zoekt-webserver has a large stable heap size (10s of gigs), and as such the
+# default GOGC=100 could be better tuned. https://dave.cheney.net/tag/gogc
+ENV GOGC=50
+
+ENTRYPOINT ["/sbin/tini", "--"]

Dockerfile.indexserver

@@ -0,0 +1,24 @@
+FROM alpine
+
+RUN apk add --no-cache ca-certificates bind-tools
+
+# Run as non-root user sourcegraph. External volumes should be mounted under /data (which will be owned by sourcegraph).
+RUN mkdir -p /home/sourcegraph
+RUN addgroup -S sourcegraph && adduser -S -G sourcegraph -h /home/sourcegraph sourcegraph && mkdir -p /data && chown -R sourcegraph:sourcegraph /data
+USER sourcegraph
+WORKDIR /home/sourcegraph
+
+ENV SRC_FRONTEND_INTERNAL http://sourcegraph-frontend-internal
+ENV DATA_DIR /data/index
+RUN mkdir -p ${DATA_DIR}
+
+# We copy from the locally built zoekt image
+COPY --from=zoekt /.ctags.d /
+COPY --from=zoekt /usr/local/bin/universal-* /usr/local/bin/zoekt-sourcegraph-indexserver /usr/local/bin/zoekt-archive-index /usr/local/bin/
+
+# zoekt indexing has a large stable heap size (gigs), and as such the default
+# GOGC=100 could be better tuned. https://dave.cheney.net/tag/gogc
+ENV GOGC=50
+
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD zoekt-sourcegraph-indexserver -index $DATA_DIR -sourcegraph_url $SRC_FRONTEND_INTERNAL -listen :6072 -interval 1m -cpu_fraction 1.0

Dockerfile.webserver

@@ -0,0 +1,22 @@
+FROM alpine
+
+RUN apk add --no-cache ca-certificates bind-tools
+
+# Run as non-root user sourcegraph. External volumes should be mounted under /data (which will be owned by sourcegraph).
+RUN mkdir -p /home/sourcegraph
+RUN addgroup -S sourcegraph && adduser -S -G sourcegraph -h /home/sourcegraph sourcegraph && mkdir -p /data && chown -R sourcegraph:sourcegraph /data
+USER sourcegraph
+WORKDIR /home/sourcegraph
+
+ENV DATA_DIR /data/index
+RUN mkdir -p ${DATA_DIR}
+
+# We copy from the locally built zoekt image
+COPY --from=zoekt /usr/local/bin/zoekt-webserver /usr/local/bin/
+
+# zoekt-webserver has a large stable heap size (10s of gigs), and as such the
+# default GOGC=100 could be better tuned. https://dave.cheney.net/tag/gogc
+ENV GOGC=50
+
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD zoekt-webserver -index $DATA_DIR -pprof -rpc