deployment and config updates

gleanerio · Jul 11, 2021 · f63c787 · f63c787
1 parent 152eb62
commit f63c787
Show file tree

Hide file tree

Showing 10 changed files with 318 additions and 0 deletions.
diff --git a/deployments/docker-base.yml → deployment/Morgue/docker-base.yml b/deployments/docker-base.yml → deployment/Morgue/docker-base.yml
diff --git a/deployments/docker-dev.yml → deployment/Morgue/docker-dev.yml b/deployments/docker-dev.yml → deployment/Morgue/docker-dev.yml
diff --git a/deployments/docker-headlessonly.yml → deployment/Morgue/docker-headlessonly.yml b/deployments/docker-headlessonly.yml → deployment/Morgue/docker-headlessonly.yml
diff --git a/deployment/gleaner-DS-APIs.yml b/deployment/gleaner-DS-APIs.yml
@@ -0,0 +1,54 @@
+version: '3'
+
+services:
+  mercantile:
+    image: fils/mercantile:latest
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.mercantile.entrypoints=http"
+      - "traefik.http.routers.mercantile.rule=Host(`api.foo.org`)"
+        #- "traefik.http.routers.mercantile.rule=(Host(`foo.org`) && Path(`/query`)"
+      - "traefik.http.middlewares.mercantile-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.mercantile.middlewares=mercantile-https-redirect"
+      - "traefik.http.routers.mercantile-secure.entrypoints=https"
+      - "traefik.http.routers.mercantile-secure.rule=Host(`api.foo.org`)"
+        #- "traefik.http.routers.mercantile-secure.rule=(Host(`foo.org`) && Path(`/query`)"
+      - "traefik.http.routers.mercantile-secure.tls=true"
+      - "traefik.http.routers.mercantile-secure.tls.certresolver=http"
+      - "traefik.http.routers.mercantile-secure.service=mercantile"
+      - "traefik.http.middlewares.mercantile-secure.headers.accesscontrolallowmethods=GET,OPTIONS,PUT,POST"
+      - "traefik.http.middlewares.mercantile-secure.headers.accesscontrolalloworigin=*"
+      - "traefik.http.middlewares.mercantile-secure.headers.accesscontrolmaxage=200"
+      - "traefik.http.middlewares.mercantile-secure.headers.addvaryheader=true"
+      - "traefik.http.middlewares.mercantile-secure.headers.accesscontrolallowcredentials=true"
+      - "traefik.http.middlewares.mercantile-secure.headers.accesscontrolallowheaders=*"
+        #- "traefik.http.middlewares.mercantile-secure.headers.accesscontrolallowheaders=Authorization,Origin,Content-Type,Accept"
+      - "traefik.http.middlewares.mercantile-secure.headers.customresponseheaders.Access-Control-Allow-Headers=*"
+        #- "traefik.http.middlewares.mercantile-secure.headers.customresponseheaders.Access-Control-Allow-Headers=hx-target,hx-request,Authorization,Origin,Content-Type,Accept"
+      - "traefik.http.routers.mercantile-secure.middlewares=mercantile-secure@docker" 
+      - "traefik.http.services.mercantile.loadbalancer.server.port=8080"
+      - "traefik.docker.network=traefik_default"
+    networks:
+      - traefik_default
+
+  koop:
+    image: fils/koop:1.0
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.koop.entrypoints=http"
+      - "traefik.http.routers.koop.rule=Host(`spatial.foo.org`)"
+      - "traefik.http.middlewares.koop-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.koop.middlewares=koop-https-redirect"
+      - "traefik.http.routers.koop-secure.entrypoints=https"
+      - "traefik.http.routers.koop-secure.rule=Host(`spatial.foo.org`)"
+      - "traefik.http.routers.koop-secure.tls=true"
+      - "traefik.http.routers.koop-secure.tls.certresolver=http"
+      - "traefik.http.routers.koop-secure.service=koop"
+      - "traefik.http.services.koop.loadbalancer.server.port=9000"
+      - "traefik.docker.network=traefik_default"
+    networks:
+      - traefik_default
+
+networks:
+  traefik_default:
+
diff --git a/deployment/gleaner-DS-NoRouter.yml b/deployment/gleaner-DS-NoRouter.yml
@@ -0,0 +1,48 @@
+version: '3'
+
+# ${MINIO_ACCESS_KEY}  
+# ${MINIO_SECRET_KEY}
+# ${GLEANER_OBJECTS}
+# ${GLEANER_GRAPH}
+
+services:
+  triplestore:
+    image: nawer/blazegraph 
+    environment:
+      JAVA_XMS: 2g
+      JAVA_XMX: 8g
+      JAVA_OPTS: -Xmx6g -Xms2g --XX:+UseG1GC
+    ports:
+      - 9999:9999
+    volumes:
+      - ${GLEANER_GRAPH}:/var/lib/blazegraph
+    networks:
+      - traefik_default
+
+  s3system:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+    volumes:
+      - ${GLEANER_OBJECTS}:/data
+    environment:
+      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
+      - MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
+    networks:
+      - traefik_default
+    command: ["server", "/data"]
+
+  headless:
+    image: chromedp/headless-shell:latest
+    labels:
+      - "traefik.backend=headlesschrome"
+      - "traefik.port=9222"
+    ports:
+      - 9222:9222
+    environment:
+     - SERVICE_PORTS=9222
+    networks:
+      - traefik_default    
+
+networks:
+  traefik_default:
diff --git a/deployment/gleaner-DS-traefik.yml b/deployment/gleaner-DS-traefik.yml
@@ -0,0 +1,138 @@
+version: '3'
+
+# ${GLEANER_ADMIN_DOMAIN}
+# ${GLEANER_OSS_DOMAIN}
+# ${GLEANER_GRAPH_DOMAIN}
+# ${GLEANER_WEB_DOMAIN}  
+# ${GLEANER_WEB2_DOMAIN}
+# ${MINIO_ACCESS_KEY}  
+# ${MINIO_SECRET_KEY}
+#
+# ${GLEANER_TRAEFIK}
+# ${GLEANER_OBJECTS}
+# ${GLEANER_GRAPH}
+
+services:
+  traefik:
+    image: traefik:v2.0.2
+    container_name: traefik
+    restart: unless-stopped
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - traefik_default
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${GLEANER_TRAEFIK}/logs:/logs
+      - ${GLEANER_TRAEFIK}/traefik.yml:/traefik.yml:ro
+      - ${GLEANER_TRAEFIK}/acme/acme.json:/acme.json
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.traefik.entrypoints=http"
+      - "traefik.http.routers.traefik.rule=Host(`${GLEANER_ADMIN_DOMAIN}`)"
+      - "traefik.http.middlewares.traefik-auth.basicauth.users=admin:SpiffyAdminPasswordHere"
+      - "traefik.http.middlewares.traefik-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.traefik.middlewares=traefik-https-redirect"
+      - "traefik.http.routers.traefik-secure.entrypoints=https"
+      - "traefik.http.routers.traefik-secure.rule=Host(`${GLEANER_ADMIN_DOMAIN}`)"
+      - "traefik.http.routers.traefik-secure.middlewares=traefik-auth"
+      - "traefik.http.routers.traefik-secure.tls=true"
+      - "traefik.http.routers.traefik-secure.tls.certresolver=http"
+      - "traefik.http.routers.traefik-secure.service=api@internal"
+
+  triplestore:
+    image: nawer/blazegraph 
+    environment:
+      JAVA_XMS: 2g
+      JAVA_XMX: 8g
+      JAVA_OPTS: -Xmx6g -Xms2g --XX:+UseG1GC
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.triplestore.entrypoints=http"
+      - "traefik.http.routers.triplestore.rule=Host(`${GLEANER_GRAPH_DOMAIN}`)"
+      - "traefik.http.middlewares.triplestore-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.triplestore.middlewares=triplestore-https-redirect"
+      - "traefik.http.routers.triplestore-secure.entrypoints=https"
+      - "traefik.http.routers.triplestore-secure.rule=Host(`${GLEANER_GRAPH_DOMAIN}`)"
+      - "traefik.http.routers.triplestore-secure.tls=true"
+      - "traefik.http.routers.triplestore-secure.tls.certresolver=http"
+      - "traefik.http.routers.triplestore-secure.service=triplestore"
+      - "traefik.http.middlewares.triplestore-secure.headers.accesscontrolallowmethods=GET,OPTIONS,PUT,POST"
+      - "traefik.http.middlewares.triplestore-secure.headers.accesscontrolalloworigin=*"
+      - "traefik.http.middlewares.triplestore-secure.headers.accesscontrolmaxage=200"
+      - "traefik.http.middlewares.triplestore-secure.headers.addvaryheader=true"
+      - "traefik.http.middlewares.triplestore-secure.headers.accesscontrolallowcredentials=true"
+      - "traefik.http.middlewares.triplestore-secure.headers.accesscontrolallowheaders=Authorization,Origin,Content-Type,Accept"
+      - "traefik.http.middlewares.triplestore-secure.headers.customresponseheaders.Access-Control-Allow-Headers=Authorization,Origin,Content-Type,Accept"
+      - "traefik.http.routers.triplestore-secure.middlewares=triplestore-secure@docker" 
+      - "traefik.http.services.triplestore.loadbalancer.server.port=9999"
+      - "traefik.docker.network=traefik_default"
+    volumes:
+      - ${GLEANER_GRAPH}:/var/lib/blazegraph
+    networks:
+      - traefik_default
+
+  s3system:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.s3system.entrypoints=http"
+      - "traefik.http.routers.s3system.rule=Host(`${GLEANER_OSS_DOMAIN}`)"
+      - "traefik.http.middlewares.s3system-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.s3system.middlewares=s3system-https-redirect"
+      - "traefik.http.routers.s3system-secure.entrypoints=https"
+      - "traefik.http.routers.s3system-secure.rule=Host(`${GLEANER_OSS_DOMAIN}`)"
+      - "traefik.http.routers.s3system-secure.tls=true"
+      - "traefik.http.routers.s3system-secure.tls.certresolver=http"
+      - "traefik.http.routers.s3system-secure.service=s3system"
+      - "traefik.http.services.s3system.loadbalancer.server.port=9000"
+      - "traefik.docker.network=traefik_default"
+    volumes:
+      - ${GLEANER_OBJECTS}:/data
+    environment:
+      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
+      - MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
+    networks:
+      - traefik_default
+    command: ["server", "/data"]
+
+  features:
+    image: fils/grow-general:latest
+    environment:
+      - S3ADDRESS=s3system:9000
+      - S3BUCKET=sites
+      - S3PREFIX=domain
+      - DOMAIN=https://${GLEANER_WEB_DOMAIN}/
+      - S3KEY=${MINIO_ACCESS_KEY}
+      - S3SECRET=${MINIO_SECRET_KEY}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.features.entrypoints=http"
+      - "traefik.http.routers.features.rule=Host(`${GLEANER_WEB_DOMAIN}`, `${GLEANER_WEB2_DOMAIN}`)"
+      - "traefik.http.middlewares.features-https-redirect.redirectscheme.scheme=https"
+      - "traefik.http.routers.features.middlewares=features-https-redirect"
+      - "traefik.http.routers.features-secure.entrypoints=https"
+      - "traefik.http.routers.features-secure.rule=Host(`${GLEANER_WEB_DOMAIN}`,`${GLEANER_WEB2_DOMAIN}`)"
+      - "traefik.http.routers.features-secure.tls=true"
+      - "traefik.http.routers.features-secure.tls.certresolver=http"
+      - "traefik.http.routers.features-secure.service=features"
+      - "traefik.http.services.features.loadbalancer.server.port=8080"
+      - "traefik.docker.network=traefik_default"
+      - "traefik.http.middlewares.features.headers.accesscontrolallowmethods=GET,OPTIONS,PUT,POST"
+      - "traefik.http.middlewares.features.headers.accesscontrolalloworigin=*"
+      - "traefik.http.middlewares.features.headers.accesscontrolmaxage=100"
+      - "traefik.http.middlewares.features.headers.addvaryheader=true"
+      - "traefik.http.middlewares.features-secure.headers.accesscontrolallowheaders=*"
+      - "traefik.http.middlewares.features-secure.headers.customresponseheaders.Access-Control-Allow-Headers=*"
+    networks:
+      - traefik_default
+
+networks:
+  traefik_default:
+
diff --git a/deployment/gleaner-IS-NoS3.yml b/deployment/gleaner-IS-NoS3.yml
@@ -0,0 +1,18 @@
+version: '3'
+
+services:
+  headless:
+    image: chromedp/headless-shell:latest
+    labels:
+      - "traefik.backend=headlesschrome"
+      - "traefik.port=9222"
+    ports:
+      - 9222:9222
+    environment:
+     - SERVICE_PORTS=9222
+    networks:
+     - traefik_default
+
+networks:
+  traefik_default:
+
diff --git a/deployment/gleaner-IS.yml b/deployment/gleaner-IS.yml
@@ -0,0 +1,48 @@
+version: '3'
+
+# ${MINIO_ACCESS_KEY}  
+# ${MINIO_SECRET_KEY}
+# ${GLEANER_OBJECTS}
+# ${GLEANER_GRAPH}
+
+services:
+  triplestore:
+    image: nawer/blazegraph 
+    environment:
+      JAVA_XMS: 2g
+      JAVA_XMX: 8g
+      JAVA_OPTS: -Xmx6g -Xms2g --XX:+UseG1GC
+    ports:
+      - 9999:9999
+    volumes:
+      - ${GLEANER_GRAPH}:/var/lib/blazegraph
+    networks:
+      - traefik_default
+
+  s3system:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+    volumes:
+      - ${GLEANER_OBJECTS}:/data
+    environment:
+      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
+      - MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
+    networks:
+      - traefik_default
+    command: ["server", "/data"]
+
+  headless:
+    image: chromedp/headless-shell:latest
+    labels:
+      - "traefik.backend=headlesschrome"
+      - "traefik.port=9222"
+    ports:
+      - 9222:9222
+    environment:
+     - SERVICE_PORTS=9222
+    networks:
+      - traefik_default    
+
+networks:
+  traefik_default:
diff --git a/configs/setenv.sh → deployment/setenv.sh b/configs/setenv.sh → deployment/setenv.sh
diff --git a/deployment/setenvIS.sh b/deployment/setenvIS.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Object store keys
+export MINIO_ACCESS_KEY=worldsbestaccesskey
+export MINIO_SECRET_KEY=worldsbestsecretkey
+
+# local data volumes
+export GLEANER_BASE=/tmp/gleaner/
+mkdir --parents ${GLEANER_BASE}
+export GLEANER_OBJECTS=${GLEANER_BASE}/datavol/s3
+export GLEANER_GRAPH=${GLEANER_BASE}/datavol/graph
+