add helpers.sh settings.sh stop.sh to make keycloak and quickstart.sh…

… work better on apple silicon and be more readable

docker-compose.yml

@@ -2,6 +2,7 @@ version: '3'
 services:
 
   blacklight:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     container_name: blacklight
     build:
       context: ./blacklight
@@ -21,8 +22,9 @@ services:
     #  - jaeger
 
   mysql:
+    platform: ${DOCKER_AMD64_PLATFORM}
     container_name: mysql
-    image: amd64/mysql:5.7.40
+    image: amd64/mysql:5.7.41
     ports:
       - 3306:3306
     environment:
@@ -35,6 +37,7 @@ services:
       timeout: 30s
 
   smui:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     container_name: smui
     build:
       context: ./smui
@@ -62,6 +65,7 @@ services:
       - mysql
 
   solr1:
+    platform: ${DOCKER_DEFAULT_PLATFORM2}
     container_name: solr1
     build: ./solr
     ports:
@@ -83,6 +87,7 @@ services:
       - keycloak
 
   solr2:
+    platform: ${DOCKER_DEFAULT_PLATFORM2}
     container_name: solr2
     build: ./solr
     ports:
@@ -104,6 +109,7 @@ services:
       - keycloak
 
   solr3:
+    platform: ${DOCKER_DEFAULT_PLATFORM2}
     container_name: solr3
     build: ./solr
     ports:
@@ -125,6 +131,7 @@ services:
       - keycloak
 
   zoo1:
+    platform: ${DOCKER_AMD64_PLATFORM}
     image: zookeeper:3.7.0
     container_name: zoo1
     hostname: zoo1
@@ -139,6 +146,7 @@ services:
 
 
   zoo2:
+    platform: ${DOCKER_AMD64_PLATFORM}
     image: zookeeper:3.7.0
     container_name: zoo2
     hostname: zoo2
@@ -153,6 +161,7 @@ services:
 
 
   zoo3:
+    platform: ${DOCKER_AMD64_PLATFORM}
     image: zookeeper:3.7.0
     container_name: zoo3
     hostname: zoo3
@@ -166,12 +175,14 @@ services:
       ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true"
 
   redis:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     container_name: quepid_redis
     image: redis:6.2.7-alpine
     ports:
       - 6379:6379
 
   quepid:
+    platform: ${DOCKER_AMD64_PLATFORM}
     container_name: quepid
     image: o19s/quepid:6.14.0
     ports:
@@ -198,6 +209,7 @@ services:
       - KEYCLOAK_REALM=chorus
       - KEYCLOAK_SITE=http://keycloak:9080
       - SIGNUP_ENABLED=true
+      - DISABLE_DATABASE_ENVIRONMENT_CHECK=0
     links:
       - mysql
       - redis
@@ -206,6 +218,7 @@ services:
       - redis
 
   rre:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     container_name: rre
     build:
       context: ./rre
@@ -220,8 +233,8 @@ services:
     depends_on:
       - solr1
 
-
   embeddings:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     container_name: embeddings
     build:
       context: ./embeddings
@@ -231,8 +244,6 @@ services:
     ports:
       - "8000:8000"
 
-
-
   solr-exporter:
     image: solr:9.1.1
     container_name: solr-exporter
@@ -249,6 +260,7 @@ services:
       - solr3
 
   prometheus:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     image: prom/prometheus:v2.32.1
     container_name: prometheus
     restart: always
@@ -265,6 +277,7 @@ services:
       - ./prometheus/:/etc/prometheus/
 
   grafana:
+    platform: ${DOCKER_DEFAULT_PLATFORM}
     image: grafana/grafana:7.5.17
     container_name: grafana
     restart: unless-stopped
@@ -284,7 +297,8 @@ services:
       - prometheus
 
   jaeger:
-    image: jaegertracing/all-in-one:1.39
+    platform: ${DOCKER_DEFAULT_PLATFORM}
+    image: jaegertracing/all-in-one:1.42
     container_name: jaeger
     restart: always
     hostname: jaegertracing

quickstart.sh

@@ -8,14 +8,6 @@ set -e
 
 log_awesome "Thank you for trying Chorus! Welcome!"
 
-
-offline_lab=false
-observability=false
-local_deploy=true
-vector_search=false
-active_search_management=false
-shutdown=false
-
 while [ ! $# -eq 0 ]
 do
 	case "$1" in
@@ -37,12 +29,17 @@ do
 			;;
 	  --with-vector-search | -vector)
   		vector_search=true
-      echo -e "${MAJOR}Configuring Chorus with vector search services enabled${RESET}"
+      log_major "Configuring Chorus with vector search services enabled"
   		;;
     --with-active-search-management | -active)
   		active_search_management=true
-      echo -e "${MAJOR}Configuring Chorus with active search management enabled${RESET}"
+      log_major "Configuring Chorus with active search management enabled"
   		;;
+    --with-apple-silicon | -apple)
+      export DOCKER_DEFAULT_PLATFORM=linux/arm64
+      export DOCKER_DEFAULT_PLATFORM2=linux/arm64/v8
+      log_major "Configuring Chorus for Apple silicon"
+      ;;
     --shutdown | -s)
 			shutdown=true
       log_major "Shutting down Chorus"
@@ -132,22 +129,22 @@ curl --user solr:SolrRocks -X POST http://localhost:8983/api/collections -H 'Con
 
 if $vector_search; then
   # Populating product data for vector search
-  echo -e "${MAJOR}Populating products for vector search, please give it a few minutes!${RESET}"
+  log_major "Populating products for vector search, please give it a few minutes!"
   ./solr/index-vectors.sh
 else
   # Populating product data for non-vector search
   if [ ! -f ./solr/data/icecat-products-150k-20200809.tar.gz ]; then
-    echo -e "${MAJOR}Downloading the sample product data.${RESET}"
-    curl --progress-bar -o ./solr/data/icecat-products-150k-20200809.tar.gz -k https://querqy.org/datasets/icecat/icecat-products-150k-20200809.tar.gz
+    log_major "Downloading the sample product data."
+    retry_until_command_success_and_responseHeader_status_is_zero "curl --progress-bar -o ./solr/data/icecat-products-150k-20200809.tar.gz -k https://querqy.org/datasets/icecat/icecat-products-150k-20200809.tar.gz"
   fi
-  echo -e "${MAJOR}Populating products, please give it a few minutes!${RESET}"
-  tar xzf ./solr/data/icecat-products-150k-20200809.tar.gz --to-stdout | curl --user solr:SolrRocks 'http://localhost:8983/solr/ecommerce/update?commit=true' --data-binary @- -H 'Content-type:application/json'
+  log_major "Populating products, please give it a few minutes!"
+  tar xzf ./solr/data/icecat-products-150k-20200809.tar.gz --to-stdout | curl --user $SOLR_USER:$SOLR_PASS 'http://localhost:8983/solr/ecommerce/update?commit=true' --data-binary @- -H 'Content-type:application/json'
 fi
 
 # Embedding service for vector search
-echo -e "${MAJOR}Preparing embeddings rewriter.${RESET}"
+log_major "Preparing embeddings rewriter."
 
-curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/querqy/rewriter/embtxt?action=save -H 'Content-type:application/json'  -d '{
+curl --user $SOLR_USER:$SOLR_PASS -X POST http://localhost:8983/solr/ecommerce/querqy/rewriter/embtxt?action=save -H 'Content-type:application/json' -d '{
   "class": "querqy.solr.embeddings.SolrEmbeddingsRewriterFactory",
          "config": {
              "model" : {
@@ -159,7 +156,7 @@ curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/querqy/r
          }
 }'
 
-curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/querqy/rewriter/embimg?action=save -H 'Content-type:application/json'  -d '{
+curl --user $SOLR_USER:$SOLR_PASS -X POST http://localhost:8983/solr/ecommerce/querqy/rewriter/embimg?action=save -H 'Content-type:application/json' -d '{
   "class": "querqy.solr.embeddings.SolrEmbeddingsRewriterFactory",
          "config": {
              "model" : {
@@ -171,9 +168,8 @@ curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/querqy/r
          }
 }'
 
-
-echo -e "${MAJOR}Defining relevancy algorithms using ParamSets.${RESET}"
-curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/config/params -H 'Content-type:application/json'  -d '{
+log_major "Defining relevancy algorithms using ParamSets."
+curl --user $SOLR_USER:$SOLR_PASS -X POST http://localhost:8983/solr/ecommerce/config/params -H 'Content-type:application/json' -d '{
   "set": {
     "visible_products":{
       "fq":["price:*", "-img_500x500:\"\""]
@@ -259,7 +255,7 @@ curl --user solr:SolrRocks -X POST http://localhost:8983/solr/ecommerce/config/p
 
 
 if $active_search_management; then
-  echo -e "${MAJOR}Setting up SMUI${RESET}"
+  log_major "Setting up SMUI"
   SOLR_INDEX_ID=`curl -S -X PUT -H "Content-Type: application/json" -d '{"name":"ecommerce", "description":"Chorus Webshop"}' http://localhost:9000/api/v1/solr-index | jq -r .returnId`
   curl -S -X PUT -H "Content-Type: application/json" -d '{"name":"product_type"}' http://localhost:9000/api/v1/${SOLR_INDEX_ID}/suggested-solr-field
   curl -S -X PUT -H "Content-Type: application/json" -d '{"name":"title"}' http://localhost:9000/api/v1/${SOLR_INDEX_ID}/suggested-solr-field
@@ -272,7 +268,7 @@ if $offline_lab; then
 
   docker-compose run --rm quepid bin/rake db:setup
   docker-compose run quepid thor user:create -a admin@choruselectronics.com "Chorus Admin" password
-  echo -e "${MINOR}Setting up Chorus Baseline Relevance case${RESET}"
+  log_minor "Setting up Chorus Baseline Relevance case"
   docker-compose run quepid thor case:create "Chorus Baseline Relevance" solr http://localhost:8983/solr/ecommerce/select JSONP "id:id, title:title, thumb:img_500x500, name, brand, product_type" "q=#\$query##&useParams=visible_products,querqy_algo" nDCG@10 admin@choruselectronics.com
   docker cp ./katas/Broad_Query_Set_rated.csv quepid:/srv/app/Broad_Query_Set_rated.csv
   docker exec quepid thor ratings:import 1 /srv/app/Broad_Query_Set_rated.csv >> /dev/null

settings.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# This script is sourced by quickstart.sh and quickstop.sh
+export DOCKER_SCAN_SUGGEST=false
+export DOCKER_DEFAULT_PLATFORM=linux/amd64
+export DOCKER_DEFAULT_PLATFORM2=linux/amd64
+export DOCKER_AMD64_PLATFORM=linux/amd64
+
+offline_lab=false
+observability=false
+local_deploy=true
+vector_search=false
+active_search_management=false
+shutdown=false
+apple_silicon=false
+
+SOLR_USER=solr
+SOLR_PASS=SolrRocks

solr/index-vectors.sh

@@ -1,27 +1,22 @@
 #!/bin/bash
 
+source settings.sh # settings.sh is located in the chorus root directory, and since this script is called from there, no "../" prefix here is necessary 
+source helpers.sh # helpers.sh, ditto
+
 DATA_DIR="./solr/data"
+S3_PRODUCT_VECTORS_LOCATION="https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-02-08"
 
-if [ ! -f ./solr/data/products-vectors-1.json ]; then
-	echo -e "${MAJOR}Downloading the products-vectors-1.json.${RESET}"
-	curl --progress-bar -o ./solr/data/products-vectors-1.json -k https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-02-08/products-vectors-1.json
-fi
-if [ ! -f ./solr/data/products-vectors-2.json ]; then
-	echo -e "${MAJOR}Downloading the products-vectors-2.json.${RESET}"
-	curl --progress-bar -o ./solr/data/products-vectors-2.json -k https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-02-08/products-vectors-2.json
-fi
-if [ ! -f ./solr/data/products-vectors-3.json ]; then
-	echo -e "${MAJOR}Downloading the products-vectors-3.json.${RESET}"
-	curl --progress-bar -o ./solr/data/products-vectors-3.json -k https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-02-08/products-vectors-3.json
-fi
-if [ ! -f ./solr/data/products-vectors-4.json ]; then
-	echo -e "${MAJOR}Downloading the products-vectors-4.json.${RESET}"
-	curl --progress-bar -o ./solr/data/products-vectors-4.json -k https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-02-08/products-vectors-4.json
-fi
+for i in {1..4}
+do
+	if [ ! -f $DATA_DIR/products-vectors-$i.json ]; then
+		log_major "Downloading the products-vectors-${i}.json."
+		retry_until_command_success_and_responseHeader_status_is_zero "curl --progress-bar -o ${DATA_DIR}/products-vectors-${i}.json -k ${S3_PRODUCT_VECTORS_LOCATION}/products-vectors-${i}.json"
+	fi
+done
 
 cd $DATA_DIR
 for f in products-vectors*.json;
-  do
-    echo "Populating products from ${f}, please give it a few minutes!"
-    curl --user solr:SolrRocks 'http://localhost:8983/solr/ecommerce/update?commit=true' --data-binary @"$f" -H 'Content-type:application/json ';
-   done;
+do
+  log_minor "Populating products from ${f}, please give it a few minutes!"
+  retry_until_command_success_and_responseHeader_status_is_zero "curl -X POST --user ${SOLR_USER}:${SOLR_PASS} 'http://localhost:8983/solr/ecommerce/update?commit=true' --data-binary @${f} -H 'Content-type:application/json'"
+done;

stop.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script stops Chorus.
+
+source settings.sh
+source helpers.sh
+
+docker-compose down -v
+if $shutdown; then
+  exit
+fi