Merge branch 'main' into add-automatic-cert-generation

drizzentic · web-flow · commit bf3d04c59da6 · 2024-08-02T14:47:05.000+03:00
diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml
@@ -5,6 +5,8 @@ services:
     image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
+    environment:
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     volumes:
       - clickhouse-data-01:/var/lib/clickhouse/
     hostname: analytics-datastore-clickhouse-01
@@ -33,6 +35,7 @@ services:
         source: clickhouse_trace_log.xml
     networks:
       public:
+      reverse-proxy:
       default:
 
   analytics-datastore-clickhouse-02:
@@ -44,6 +47,8 @@ services:
           - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
     ulimits:
       noFile: 262144
+    environment:
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     volumes:
       - clickhouse-data-02:/var/lib/clickhouse/
     configs:
@@ -78,6 +83,8 @@ services:
           - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
     ulimits:
       noFile: 262144
+    environment:
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     volumes:
       - clickhouse-data-03:/var/lib/clickhouse/
     configs:
@@ -108,6 +115,8 @@ services:
     hostname: analytics-datastore-clickhouse-04
     ulimits:
       noFile: 262144
+    environment:
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     volumes:
       - clickhouse-data-04:/var/lib/clickhouse/
     configs:
@@ -213,4 +222,7 @@ networks:
   public:
     name: clickhouse_public
     external: true
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
   default:
diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml
@@ -5,6 +5,8 @@ services:
     image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
+    environment:
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     volumes:
       - clickhouse-data:/var/lib/clickhouse/
     configs:
@@ -18,6 +20,7 @@ services:
         source: clickhouse_trace_log.xml
     networks:
       public:
+      reverse-proxy:
       default:
 
 volumes:
@@ -49,4 +52,7 @@ networks:
   public:
     name: clickhouse_public
     external: true
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
   default:
diff --git a/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js b/analytics-datastore-clickhouse/importer/config/clickhouseConfig.js
@@ -7,11 +7,16 @@ const CLICKHOUSE_HOST =
   process.env.CLICKHOUSE_HOST || 'analytics-datastore-clickhouse';
 const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || '8123');
 const CLICKHOUSE_DEBUG = Boolean(process.env.CLICKHOUSE_DEBUG || false);
+const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || '';
 
 const clickhouse = new ClickHouse({
   url: CLICKHOUSE_HOST,
   port: CLICKHOUSE_PORT,
   debug: CLICKHOUSE_DEBUG,
+  basicAuth: {
+    username: 'default',
+    password: CLICKHOUSE_PASSWORD,
+  },
   raw: true,
 });
 
diff --git a/analytics-datastore-clickhouse/importer/docker-compose.config.yml b/analytics-datastore-clickhouse/importer/docker-compose.config.yml
@@ -9,6 +9,7 @@ services:
     environment:
       CLICKHOUSE_HOST: ${CLICKHOUSE_HOST}
       CLICKHOUSE_PORT: ${CLICKHOUSE_PORT}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
       CLUSTERED_MODE: ${CLUSTERED_MODE}
     configs:
       - source: config-importer-clickhouseConfig.js
diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json
@@ -8,6 +8,7 @@
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
     "CLICKHOUSE_PORT": "8123",
+    "CLICKHOUSE_PASSWORD": "dev_password_only",
     "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
     "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
     "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
diff --git a/analytics-datastore-elastic-search/docker-compose.yml b/analytics-datastore-elastic-search/docker-compose.yml
@@ -37,7 +37,7 @@ services:
       start_period: 1m
     volumes:
       - es-data:/usr/share/elasticsearch/data
-      - /backups:/backups/elasticsearch
+      - ${ES_BACKUPS_PATH:-/backups}:/backups/elasticsearch
     networks:
       public:
 
diff --git a/dashboard-visualiser-superset/importer/config/superset-export.zip b/dashboard-visualiser-superset/importer/config/superset-export.zip
diff --git a/dashboard-visualiser-superset/importer/config/supersetConfig.js b/dashboard-visualiser-superset/importer/config/supersetConfig.js
@@ -14,6 +14,8 @@ const SUPERSET_LOGIN_PATH =
   process.env.SUPERSET_LOGIN_PATH || '/api/v1/security/login';
 const SUPERSET_IMPORT_PATH =
   process.env.SUPERSET_IMPORT_PATH || '/api/v1/assets/import/';
+const SUPERSET_DATABASE_PUT_PATH =
+  process.env.SUPERSET_DATABASE_PUT_PATH || '/api/v1/database';
 const CONFIG_FILE = process.env.CONFIG_FILE;
 const SUPERSET_SSL = process.env.SUPERSET_SSL || 'false';
 
@@ -27,7 +29,7 @@ const getAccessToken = async () => {
     refresh: true,
   });
 
-  var config = {
+  const config = {
     method: 'POST',
     url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_LOGIN_PATH}`,
     headers: {
@@ -47,39 +49,81 @@ const getAccessToken = async () => {
   }
 };
 
+const importZipConfig = async (accessToken) => {
+  const data = new FormData();
+
+  if (CONFIG_FILE) {
+    data.append(
+      'bundle',
+      fs.createReadStream(path.resolve(__dirname, CONFIG_FILE))
+    );
+
+    const config = {
+      method: 'POST',
+      url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_IMPORT_PATH}`,
+      headers: {
+        'Content-Type': 'application/zip',
+        Authorization: `Bearer ${accessToken}`,
+        ...data.getHeaders(),
+      },
+      data: data,
+    };
+
+    const res = await axios(config);
+
+    console.log('\n', res.data);
+    console.log('\nConfig imported successfully. exit.');
+  } else {
+    throw new Error(
+      '\nNo path was provided. Please provide the path of the config.'
+    );
+  }
+}
+
+const replaceClickhouseConnectionString = async (accessToken) => {
+  const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || 'analytics-datastore-clickhouse';
+  const CLICKHOUSE_PORT = process.env.CLICKHOUSE_PORT || '8123';
+  const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || 'dev_password_only';
+
+  const databaseConfig = {
+    allow_ctas: false,
+    allow_cvas: false,
+    allow_dml: false,
+    allow_file_upload: false,
+    allow_run_async: false,
+    cache_timeout: 0,
+    configuration_method: "sqlalchemy_form",
+    database_name: "Clickhouse connection",
+    driver: "connect",
+    engine: "clickhousedb",
+    expose_in_sqllab: true,
+    sqlalchemy_uri: `clickhousedb+connect://default:${CLICKHOUSE_PASSWORD}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/default`,
+    uuid: "868ecd6d-f099-46ab-a100-dd91173bc63f"
+  };
+
+  const config = {
+    method: 'POST',
+    url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_DATABASE_PUT_PATH}`,
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${accessToken}`
+    },
+    data: databaseConfig,
+  };
+
+  const res = await axios(config);
+
+  console.log('\n', res.data);
+  console.log('\Database connection updated successfully. exit.');
+}
+
 (async () => {
   try {
     const accessToken = await getAccessToken();
 
     if (accessToken) {
-      var data = new FormData();
-
-      if (CONFIG_FILE) {
-        data.append(
-          'bundle',
-          fs.createReadStream(path.resolve(__dirname, CONFIG_FILE))
-        );
-
-        var config = {
-          method: 'POST',
-          url: `${protocol}://${SUPERSET_SERVICE_NAME}:${SUPERSET_API_PORT}${SUPERSET_IMPORT_PATH}`,
-          headers: {
-            'Content-Type': 'application/zip',
-            Authorization: `Bearer ${accessToken}`,
-            ...data.getHeaders(),
-          },
-          data: data,
-        };
-
-        const res = await axios(config);
-
-        console.log('\n', res.data);
-        console.log('\nConfig imported successfully. exit.');
-      } else {
-        throw new Error(
-          '\nNo path was provided. Please provide the path of the config.'
-        );
-      }
+      await importZipConfig(accessToken);
+      await replaceClickhouseConnectionString(accessToken);
     } else {
       throw new Error('\nNo access token was generated.');
     }
diff --git a/dashboard-visualiser-superset/importer/docker-compose.config.yml b/dashboard-visualiser-superset/importer/docker-compose.config.yml
@@ -13,12 +13,17 @@ services:
       SUPERSET_API_USERNAME: ${SUPERSET_USERNAME}
       CONFIG_FILE: ${SUPERSET_CONFIG_FILE}
       SUPERSET_SSL: ${SUPERSET_SSL}
+      CLICKHOUSE_HOST: ${CLICKHOUSE_HOST}
+      CLICKHOUSE_PORT: ${CLICKHOUSE_PORT}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
     configs:
       - source: config-importer-supersetConfig.js
         target: /supersetConfig.js
       - source: config-importer-superset-export.zip
         target: /superset-export.zip
     command: sh -c "cd / && npm i axios form-data && node /supersetConfig.js"
+    networks:
+      clickhouse:
 
 configs:
   config-importer-supersetConfig.js:
@@ -31,3 +36,8 @@ configs:
     name: config-importer-superset-export.zip-${config_importer_superset_export_zip_DIGEST:?err}
     labels:
       name: superset
+
+networks:
+  clickhouse:
+    name: clickhouse_public
+    external: true
diff --git a/dashboard-visualiser-superset/package-metadata.json b/dashboard-visualiser-superset/package-metadata.json
@@ -33,6 +33,9 @@
     "SUPERSET_POSTGRESQL_PASSWORD": "instant101",
     "SUPERSET_POSTGRESQL_DATABASE": "superset",
     "SUPERSET_POSTGRESQL_URL": "postgres-1:5432",
+    "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
+    "CLICKHOUSE_PORT": 8123,
+    "CLICKHOUSE_PASSWORD": "dev_password_only",
     "SUPERSET_TRAEFIK_SUBDOMAIN": "superset"
   }
 }
diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml
@@ -18,7 +18,7 @@ services:
       replicas: 1
     volumes:
       - prometheus_data_backup:/prometheus
-      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${DOCKER_SOCK_FOLDER}:/var/run/docker.sock:ro
     configs:
       - target: /etc/prometheus/prometheus.yml
         source: prometheus.yml
diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml
@@ -77,7 +77,7 @@ services:
     user: root
     volumes:
       - prometheus-data:/prometheus
-      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${DOCKER_SOCK_FOLDER}:/var/run/docker.sock:ro
     configs:
       - target: /etc/prometheus/prometheus.yml
         source: prometheus.yml
@@ -99,7 +99,7 @@ services:
       - /:/rootfs:ro
       - /var/run:/var/run
       - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker
+      - ${DOCKER_LIB_FOLDER}:/var/lib/docker
       - /dev/disk/:/dev/disk:ro
     deploy:
       mode: global
@@ -133,7 +133,7 @@ services:
   promtail:
     image: ${PROMTAIL_IMAGE}
     volumes:
-      - /var/lib/docker/containers:/host/containers
+      - ${DOCKER_LIB_FOLDER}/containers:/host/containers
       - /var/log:/var/log:ro
     configs:
       - target: /etc/promtail/promtail-config.yml
diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json
@@ -34,6 +34,8 @@
     "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only",
     "MO_RETENTION_TIME": "15d",
     "GF_SERVER_SERVE_FROM_SUB_PATH": "false",
-    "MINIO_BROWSER_REDIRECT_URL": ""
+    "MINIO_BROWSER_REDIRECT_URL": "",
+    "DOCKER_SOCK_FOLDER": "/var/run/docker.sock",
+    "DOCKER_LIB_FOLDER": "/var/lib/docker/"
   }
 }
diff --git a/test/cucumber/features/single-mode/superset.feature b/test/cucumber/features/single-mode/superset.feature
@@ -2,19 +2,19 @@ Feature: Dashboard Visualiser Superset?
     Does the Dashboard Visualiser Superset package work as expected
 
   Scenario: Init Dashboard Visualiser Superset
-    Given I use parameters "package init -n=dashboard-visualiser-superset --only --dev --env-file=.env.local"
+    Given I use parameters "package init -n=dashboard-visualiser-superset --dev --env-file=.env.local"
     When I launch the platform with params
     Then The service "dashboard-visualiser-superset" should be started with 1 replica
     And The service "postgres-metastore" should be started with 1 replica
     And The service "superset-config-importer" should be removed
-    And The service "ddashboard-visualiser-superset" should have healthy containers
+    And The service "dashboard-visualiser-superset" should have healthy containers
     And The service "dashboard-visualiser-superset" should be connected to the networks
       | reverse-proxy_public | clickhouse_public | keycloak_public | superset_default |
     And There should be 2 service
-    And There should be 2 volumes
+    And There should be 3 volumes
 
   Scenario: Destroy Dashboard Visualiser Superset
-    Given I use parameters "package destroy -n=dashboard-visualiser-superset --only --dev --env-file=.env.local"
+    Given I use parameters "package destroy -n=dashboard-visualiser-superset --dev --env-file=.env.local"
     When I launch the platform with params
     Then The service "dashboard-visualiser-superset" should be removed
     And There should be 0 service

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,9 @@`
`33`	`33`	`"SUPERSET_POSTGRESQL_PASSWORD": "instant101",`
`34`	`34`	`"SUPERSET_POSTGRESQL_DATABASE": "superset",`
`35`	`35`	`"SUPERSET_POSTGRESQL_URL": "postgres-1:5432",`
	`36`	`+ "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",`
	`37`	`+ "CLICKHOUSE_PORT": 8123,`
	`38`	`+ "CLICKHOUSE_PASSWORD": "dev_password_only",`
`36`	`39`	`"SUPERSET_TRAEFIK_SUBDOMAIN": "superset"`
`37`	`40`	`}`
`38`	`41`	`}`