Skip to content

Commit

Permalink
Merge branch 'master' into merge-master
Browse files Browse the repository at this point in the history
  • Loading branch information
sebgl committed Jul 24, 2019
2 parents c0194f6 + 51c4be8 commit 5275ada
Show file tree
Hide file tree
Showing 44 changed files with 1,441 additions and 285 deletions.
6 changes: 4 additions & 2 deletions .ci/jobs/gke-e2e-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
artifactNumToKeep: 10
name: cloud-on-k8s-versions-gke
project-type: pipeline
triggers:
- timed: '0 0 * * 1-5'
parameters:
- string:
name: IMAGE
description: "Docker image with ECK"
pipeline-scm:
scm:
- git:
Expand Down
27 changes: 9 additions & 18 deletions build/ci/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ VAULT_GKE_CREDS_SECRET ?= secret/cloud-team/cloud-ci/ci-gcp-k8s-operator
GKE_CREDS_FILE ?= credentials.json
VAULT_PUBLIC_KEY ?= secret/release/license
PUBLIC_KEY_FILE ?= license.key
VAULT_DOCKER_CREDENTIALS ?= secret/cloud-team/cloud-ci/cloudadmin
VAULT_DOCKER_CREDENTIALS ?= secret/devops-ci/cloud-on-k8s/eckadmin
DOCKER_CREDENTIALS_FILE ?= docker_credentials.file
VAULT_AWS_CREDS ?= secret/cloud-team/cloud-ci/eck-release
VAULT_AWS_ACCESS_KEY_FILE ?= aws_access_key.file
Expand Down Expand Up @@ -48,7 +48,7 @@ vault-docker-creds:
@ VAULT_TOKEN=$(VAULT_TOKEN) \
vault read \
-address=$(VAULT_ADDR) \
-field=password \
-field=value \
$(VAULT_DOCKER_CREDENTIALS) \
> $(DOCKER_CREDENTIALS_FILE)

Expand All @@ -71,7 +71,7 @@ vault-aws-creds:

ci-pr: check-license-header
docker build -f Dockerfile -t cloud-on-k8s-ci-pr .
docker run --rm -t \
@ docker run --rm -t \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
Expand All @@ -86,7 +86,7 @@ ci-pr: check-license-header

ci-release: vault-public-key vault-docker-creds
docker build -f Dockerfile -t cloud-on-k8s-ci-release .
docker run --rm -t \
@ docker run --rm -t \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
Expand All @@ -105,7 +105,7 @@ ci-release: vault-public-key vault-docker-creds
# Will be uploaded to https://download.elastic.co/downloads/eck/$TAG_NAME/all-in-one.yaml
yaml-upload: vault-aws-creds
docker build -f Dockerfile -t cloud-on-k8s-ci-release .
docker run --rm -t \
@ docker run --rm -t \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
-e "AWS_ACCESS_KEY_ID=$(shell cat $(VAULT_AWS_ACCESS_KEY_FILE))" \
Expand All @@ -119,7 +119,7 @@ yaml-upload: vault-aws-creds
# Spawn a k8s cluster, and run e2e tests against it
ci-e2e: vault-gke-creds
docker build -f Dockerfile -t cloud-on-k8s-ci-e2e .
docker run --rm -t \
@ docker run --rm -t \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
Expand All @@ -137,7 +137,7 @@ ci-e2e: vault-gke-creds
# Run e2e tests in GKE against provided ECK image
ci-e2e-rc: vault-gke-creds
docker build -f Dockerfile -t cloud-on-k8s-ci-e2e .
docker run --rm -t \
@ docker run --rm -t \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
Expand All @@ -156,7 +156,7 @@ ci-e2e-rc: vault-gke-creds
# Remove k8s cluster
ci-e2e-delete-cluster: vault-gke-creds
docker build -f Dockerfile -t cloud-on-k8s-ci-e2e .
docker run --rm -t \
@ docker run --rm -t \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
Expand All @@ -168,7 +168,7 @@ ci-e2e-delete-cluster: vault-gke-creds

# Remove all unused resources in GKE
ci-gke-cleanup: ci-e2e-delete-cluster
docker run --rm -t \
@ docker run --rm -t \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-w $(GO_MOUNT_PATH) \
-e "GCLOUD_PROJECT=$(GCLOUD_PROJECT)" \
Expand All @@ -177,12 +177,3 @@ ci-gke-cleanup: ci-e2e-delete-cluster
cloud-on-k8s-ci-e2e \
bash -c "GKE_CLUSTER_VERSION=1.11 $(GO_MOUNT_PATH)/operators/hack/gke-cluster.sh auth && \
$(GO_MOUNT_PATH)/build/ci/delete_unused_disks.py"

# Run docs build
ci-build-docs:
docker run --rm -t \
-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
docker.elastic.co/docs/build:1 \
bash -c "git clone https://github.com/elastic/docs.git && \
/docs/build_docs.pl --doc $(GO_MOUNT_PATH)/docs/index.asciidoc --out $(GO_MOUNT_PATH)/docs/html --chunk 1 && \
test -e $(GO_MOUNT_PATH)/docs/html/index.html"
8 changes: 5 additions & 3 deletions build/ci/e2e/GKE_k8s_versions.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ pipeline {
VAULT_SECRET_ID = credentials('vault-secret-id')
REGISTRY = "eu.gcr.io"
GCLOUD_PROJECT = credentials('k8s-operators-gcloud-project')
OPERATOR_IMAGE = "${IMAGE}"
LATEST_RELEASED_IMG = "${IMAGE}"
}

stages {
Expand All @@ -26,7 +28,7 @@ pipeline {
}
steps {
checkout scm
sh 'make -C build/ci ci-e2e'
sh 'make -C build/ci ci-e2e-rc'
}
}
stage("1.12") {
Expand All @@ -39,7 +41,7 @@ pipeline {
}
steps {
checkout scm
sh 'make -C build/ci ci-e2e'
sh 'make -C build/ci ci-e2e-rc'
}
}
stage("1.13") {
Expand All @@ -52,7 +54,7 @@ pipeline {
}
steps {
checkout scm
sh 'make -C build/ci ci-e2e'
sh 'make -C build/ci ci-e2e-rc'
}
}
}
Expand Down
59 changes: 59 additions & 0 deletions build/ci/nightly/Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
pipeline {

agent {
label 'linux'
}

options {
timeout(time: 1, unit: 'HOURS')
}

environment {
VAULT_ADDR = credentials('vault-addr')
VAULT_ROLE_ID = credentials('vault-role-id')
VAULT_SECRET_ID = credentials('vault-secret-id')
GCLOUD_PROJECT = credentials('k8s-operators-gcloud-project')
REGISTRY = "push.docker.elastic.co"
REPOSITORY = "eck-snapshots"
IMG_NAME = "eck-operator"
SNAPSHOT = "true"
DOCKER_IMAGE_NO_TAG = "docker.elastic.co/${REPOSITORY}/${IMG_NAME}"
}

stages {
stage('Run unit and integration tests') {
steps {
sh 'make -C build/ci ci-pr'
}
}
stage('Build and push Docker image') {
steps {
sh """
export VERSION=\$(cat $WORKSPACE/operators/VERSION)-\$(date +%F)-\$(git rev-parse --short --verify HEAD)
export OPERATOR_IMAGE=${REGISTRY}/${REPOSITORY}/${IMG_NAME}:\$VERSION
make -C build/ci ci-release
"""
}
}
}

post {
success {
script {
def version = sh(returnStdout: true, script: 'cat $WORKSPACE/operators/VERSION')
def hash = sh(returnStdout: true, script: 'git rev-parse --short --verify HEAD')
def date = new Date()
def image = env.DOCKER_IMAGE_NO_TAG + ":" + version + "-" + date.format("yyyy-MM-dd") + "-" + hash
currentBuild.description = image

build job: 'cloud-on-k8s-versions-gke',
parameters: [string(name: 'IMAGE', value: image)],
wait: false
}
}
cleanup {
cleanWs()
}
}

}
23 changes: 10 additions & 13 deletions build/ci/pr/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,16 @@ pipeline {
}
stage("Run docs build") {
steps {
checkout scm
sh 'make -C build/ci ci-build-docs'
cleanWs()
sh 'git clone git@github.com:elastic/docs.git'
sh 'git clone git@github.com:elastic/cloud-on-k8s.git'
sh """
$WORKSPACE/docs/build_docs \
--doc $WORKSPACE/cloud-on-k8s/docs/index.asciidoc \
--out $WORKSPACE/cloud-on-k8s/docs/html \
--chunk 1
"""
sh 'test -e $WORKSPACE/cloud-on-k8s/docs/html/index.html'
}
}
stage("Run smoke E2E tests") {
Expand All @@ -61,17 +69,6 @@ pipeline {
}

post {
success {
withEnv([
'REGISTRY=push.docker.elastic.co',
'REPOSITORY=eck-snapshots',
'IMG_SUFFIX=',
'SNAPSHOT_RELEASE=true',
'TAG_NAME=${ghprbPullId}'
]) {
sh 'make -C build/ci ci-release'
}
}
cleanup {
script {
if (notOnlyDocs()) {
Expand Down
36 changes: 28 additions & 8 deletions docs/accessing-services.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ To access Elasticsearch, Kibana or APM Server, the operator manages a default us

[source,sh]
----
> kubectl get secret hulk-elastic-user -o go-template='{{.data.elastic | base64decode }}'
> kubectl get secret hulk-es-elastic-user -o go-template='{{.data.elastic | base64decode }}'
42xyz42citsale42xyz42
----

Expand All @@ -46,6 +46,7 @@ For each resource, `Elasticsearch`, `Kibana` or `ApmServer`, the operator manage
> kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
hulk-apm-http ClusterIP 10.19.212.105 <none> 8200:31000/TCP 1m
hulk-es-http ClusterIP 10.19.252.160 <none> 9200:31320/TCP 1m
hulk-kb-http ClusterIP 10.19.247.151 <none> 5601:31380/TCP 1m
----
Expand Down Expand Up @@ -76,6 +77,7 @@ spec:
> kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
hulk-apm-http ClusterIP 10.19.212.105 35.176.227.106 8200:31000/TCP 1m
hulk-es-http LoadBalancer 10.19.252.160 35.198.131.115 9200:31320/TCP 1m
hulk-kb-http LoadBalancer 10.19.247.151 35.242.197.228 5601:31380/TCP 1m
----
Expand Down Expand Up @@ -141,8 +143,9 @@ spec:
You can bring your own certificate to configure TLS to ensure that communication between HTTP clients and the cluster is encrypted.

Create a Kubernetes secret with:
. tls.crt: the certificate (or a chain).
. tls.key: the private key to the first certificate in the certificate chain.

- tls.crt: the certificate (or a chain).
- tls.key: the private key to the first certificate in the certificate chain.

[source,sh]
----
Expand All @@ -160,6 +163,23 @@ spec:
secretName: my-cert
----

[float]
[id="{p}-disable-tls"]
==== Disable TLS

You can explicitly disable TLS for Kibana or APM Server if you want to.

[source,yaml]
----
spec:
http:
tls:
selfSignedCertificate:
disabled: true
----

TLS cannot be disabled for Elasticsearch.

[float]
[id="{p}-request-elasticsearch-endpoint"]
=== Requesting the Elasticsearch endpoint
Expand All @@ -178,7 +198,7 @@ NAME=hulk
kubectl get secret "$NAME-ca" -o go-template='{{index .data "ca.pem" | base64decode }}' > ca.pem
PW=$(kubectl get secret "$NAME-elastic-user" -o go-template='{{.data.elastic | base64decode }}')
curl --cacert ca.pem -u elastic:$PW https://$NAME-es:9200/
curl --cacert ca.pem -u elastic:$PW https://$NAME-es-http:9200/
----

*Outside the Kubernetes cluster*
Expand All @@ -191,11 +211,11 @@ curl --cacert ca.pem -u elastic:$PW https://$NAME-es:9200/
----
NAME=hulk
kubectl get secret "$NAME-ca" -o go-template='{{index .data "ca.pem" | base64decode }}' > ca.pem
IP=$(kubectl get svc "$NAME-es" -o jsonpath='{.status.loadBalancer.ingress[].ip}')
PW=$(kubectl get secret "$NAME-elastic-user" -o go-template='{{.data.elastic | base64decode }}')
kubectl get secret "$NAME-es-http-certs-public" -o go-template='{{index .data "tls.crt" | base64decode }}' > tls.crt
IP=$(kubectl get svc "$NAME-es-http" -o jsonpath='{.status.loadBalancer.ingress[].ip}')
PW=$(kubectl get secret "$NAME-es-elastic-user" -o go-template='{{.data.elastic | base64decode }}')
curl --cacert ca.pem -u elastic:$PW https://$IP:9200/
curl --cacert tls.crt -u elastic:$PW https://$IP:9200/
----

Now you should get this message:
Expand Down
Loading

0 comments on commit 5275ada

Please sign in to comment.