[DOC-562] Example Cache Override (#234)

* add OVERRIDE env var to toolchain docker compose

* add override flag to arangoproxy

* add OVERRIDE injection to start_arangoproxy script

* add override flag to arangoproxy config, set the override flag in example struct creation

* add override pipeline parameter to generate job

* use whitespace as default override arg in circleci

* [skip ci] Automatic commit of generated files from CircleCI

* fix commit generate name chekc

* [skip ci] Automatic commit of generated files from CircleCI

* fix commit-generated workflow

* use error wrapper in download artifacts for commit-generated

* [skip ci] Automatic commit of generated files from CircleCI

* extend override to use regexes

* restore export OVERRIDE in circleci

* fallback override

* add override param explanation in CIRCLECI.md

* adjust readme

---------

Co-authored-by: dandimeo <daniele@arangodb.com>
Co-authored-by: Paula <paula.mihu@arangodb.com>

Author: 3 people

.circleci/commit.js

@@ -24,7 +24,7 @@ async function commit_generated() {
 
         let jobsData = await jobs.json();
         for (let job of jobsData.items) {
-            if (job.name != "build-with-generated") continue
+            if (job.name != "generate") continue
             let artifacts = await fetch('https://circleci.com/api/v2/project/gh/arangodb/docs-hugo/'+job.job_number+'/artifacts', {
                 method: 'GET',
                 headers: {'content-type': 'application/json', 'Circle-Token': process.env.CIRCLECI_API_TOKEN},

.circleci/config.yml

@@ -19,6 +19,10 @@ parameters:
     type: string
     default: ""
 
+  override:
+    type: string
+    default: ""
+
   version:
     type: string
     default: ""

.circleci/generate_config.py

@@ -264,6 +264,7 @@ def workflow_generate_launch_command(config):
 export ENV=\"circleci\"\n \
 export HUGO_URL=https://<< pipeline.parameters.deploy-url >>--docs-hugo.netlify.app\n \
 export HUGO_ENV=examples\n \
+export OVERRIDE=<< pipeline.parameters.override >>\n \
 export GENERATORS='<< parameters.generators >>'\n"
 
     for i in range(len(versions)):
@@ -321,10 +322,12 @@ def workflow_commit_generated_download_data(config):
     for i in range(len(versions)):
         version = versions[i]["name"]
         cmd = f"{cmd}\n\
+set +e\n\
 wget $base_url/{version}-generated.tar\n\
 tar -xf {version}-generated.tar -C docs-hugo/site/data/\n\
+set -e\n\
 "
-    config["commands"]["store-generated-data"]["steps"][0]["run"]["command"] = cmd
+    config["commands"]["download-generated-data"]["steps"][0]["run"]["command"] = cmd
 
     return config
 

CIRCLECI.md

@@ -93,6 +93,33 @@ arguments are invoked:
 | `deploy-url` | `deploy-preview-{PR_NUMBER}` |
 | `commit-generated` | `true` |
 
+### `cache override`
+
+You can override the cache of an example with the `override` CircleCI parameter
+in the `generate` workflow.
+
+The override parameter is a comma-separated string of regexes.
+
+The comma will be replaced by `|` and creates an `OR` of all the regexes in the
+`override` parameter.
+
+The example below overrides all examples having `http` or starting with `aql` in
+the example name. You can also specify the name of the example to override the
+cache for, i.e. `AqlDateTimeToLocal_3`. 
+
+Note that the override is valid for all versions that are specified using the
+`arangodb` parameters. You can override the example output for a single version
+or for multiple versions.
+
+| Name | Value |
+|:-----|:------|
+| `workflow` | `generate` |
+| `arangodb-3_10` | `{string in PR Template at 3.10}` |
+| `arangodb-3_11` | `{string in PR Template at 3.11}` |
+| `generators` | `examples` |
+| `commit-generated` | `true` |
+| `override` | `http,^aql.*` |
+
 ## Release workflow (ArangoDB)
 
 To run a release job for a new ArangoDB patch release (e.g. 3.11.4), follow the

site/data/3.11/cache.json

@@ -2129,7 +2129,7 @@
 	},
 	"aqlDateTimeToLocal_3_single": {
 		"request": "LS0tCm5hbWU6IGFxbERhdGVUaW1lVG9Mb2NhbF8zCmRlc2NyaXB0aW9uOiAnJwotLS0KUkVUVVJOIERBVEVfVVRDVE9MT0NBTChEQVRFX05PVygpLCAiQWZyaWNhL0xhZ29zIiwgdHJ1ZSk=",
-		"response": "eyJpbnB1dCI6IlJFVFVSTiBEQVRFX1VUQ1RPTE9DQUwoREFURV9OT1coKSwgXCJBZnJpY2EvTGFnb3NcIiwgdHJ1ZSkiLCJvdXRwdXQiOiJbb2JqZWN0IEFyYW5nb1F1ZXJ5Q3Vyc29yLCBjb3VudDogMSwgY2FjaGVkOiBmYWxzZSwgaGFzTW9yZTogZmFsc2VdXG5cblsgXG4gIHsgXG4gICAgXCJsb2NhbFwiIDogXCIyMDIzLTA5LTE5VDE2OjM4OjE5LjY3MlwiLCBcbiAgICBcInR6ZGJcIiA6IFwiMjAyM2FcIiwgXG4gICAgXCJ6b25lSW5mb1wiIDogeyBcbiAgICAgIFwibmFtZVwiIDogXCJXQVRcIiwgXG4gICAgICBcImJlZ2luXCIgOiBcIjE5MTktMDgtMzFUMjM6MzA6MDAuMDAwWlwiLCBcbiAgICAgIFwiZW5kXCIgOiBudWxsLCBcbiAgICAgIFwiZHN0XCIgOiBmYWxzZSwgXG4gICAgICBcIm9mZnNldFwiIDogMzYwMCBcbiAgICB9IFxuICB9IFxuXSIsImVycm9yIjoiIiwib3B0aW9ucyI6eyJkZXNjcmlwdGlvbiI6IiIsIm5hbWUiOiJhcWxEYXRlVGltZVRvTG9jYWxfMyIsInR5cGUiOiJzaW5nbGUiLCJyZW5kZXIiOiJpbnB1dC9vdXRwdXQifX0K"
+		"response": "eyJpbnB1dCI6IlJFVFVSTiBEQVRFX1VUQ1RPTE9DQUwoREFURV9OT1coKSwgXCJBZnJpY2EvTGFnb3NcIiwgdHJ1ZSkiLCJvdXRwdXQiOiJbIFxuICB7IFxuICAgIFwibG9jYWxcIiA6IFwiMjAyMy0wOS0yMlQxNjowNDowMC4yMDVcIiwgXG4gICAgXCJ0emRiXCIgOiBcIjIwMjNhXCIsIFxuICAgIFwiem9uZUluZm9cIiA6IHsgXG4gICAgICBcIm5hbWVcIiA6IFwiV0FUXCIsIFxuICAgICAgXCJiZWdpblwiIDogXCIxOTE5LTA4LTMxVDIzOjMwOjAwLjAwMFpcIiwgXG4gICAgICBcImVuZFwiIDogbnVsbCwgXG4gICAgICBcImRzdFwiIDogZmFsc2UsIFxuICAgICAgXCJvZmZzZXRcIiA6IDM2MDAgXG4gICAgfSBcbiAgfSBcbl0iLCJlcnJvciI6IiIsIm9wdGlvbnMiOnsiZGVzY3JpcHRpb24iOiIiLCJuYW1lIjoiYXFsRGF0ZVRpbWVUb0xvY2FsXzMiLCJ0eXBlIjoic2luZ2xlIiwicmVuZGVyIjoiaW5wdXQvb3V0cHV0In19Cg=="
 	},
 	"aqlDateTimeToUTC_1_single": {
 		"request": "LS0tCm5hbWU6IGFxbERhdGVUaW1lVG9VVENfMQpkZXNjcmlwdGlvbjogJycKLS0tClJFVFVSTiBbCiAgREFURV9MT0NBTFRPVVRDKCIyMDIwLTAzLTE1VDAwOjAwOjAwLjAwMCIsICJFdXJvcGUvQmVybGluIiksCiAgREFURV9MT0NBTFRPVVRDKCIyMDIwLTAzLTE1VDAwOjAwOjAwLjAwMCIsICJBbWVyaWNhL05ld19Zb3JrIiksCiAgREFURV9MT0NBTFRPVVRDKCIyMDIwLTAzLTE1VDAwOjAwOjAwLjAwMCIsICJVVEMiKQpd",

toolchain/arangoproxy/cmd/main.go

@@ -4,18 +4,20 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/arangodb/docs/migration-tools/arangoproxy/internal"
 	"github.com/arangodb/docs/migration-tools/arangoproxy/internal/models"
 )
 
 var configFile string
-var env string
+var env, override string
 var help, useServers bool
 
 // Pre-Run Setup
 func init() {
 	flag.StringVar(&configFile, "config", "./configs/local.yaml", "path of config file")
+	flag.StringVar(&override, "override", " ", "override examples")
 	flag.BoolVar(&help, "help", false, "Display help usage")
 	flag.BoolVar(&useServers, "use-servers", false, "Enable communication with arangodb servers")
 	flag.Parse()
@@ -25,6 +27,7 @@ func init() {
 		fmt.Printf("Error loading config: %s\n, aborting...", err.Error())
 		os.Exit(1)
 	}
+	models.Conf.Override = strings.ReplaceAll(override, ",", "|")
 
 	models.Logger.Printf(startupBanner)
 	models.Logger.Printf("./arangoproxy -help for help usage\n\n")
@@ -34,6 +37,7 @@ func init() {
 		models.Logger.Printf("Usage: ...\n")
 		os.Exit(0)
 	}
+	models.Logger.Printf("Configuration:\n%v\n", models.Conf)
 
 	models.Logger.Printf("Setup Done\n---------\n")
 

toolchain/arangoproxy/internal/models/config.go

@@ -9,7 +9,8 @@ import (
 type Config struct {
 	Repositories []Repository `yaml:"repositories"` // ArangoDB instances
 	Cache        string       `yaml:"cache"`        // Cache configuration
-	Datasets     string       `yaml:"datasetsFile"` // Logfile
+	Datasets     string       `yaml:"datasetsFile"`
+	Override     string       `yaml:"-"`
 }
 
 var Conf Config

toolchain/arangoproxy/internal/models/example.go

@@ -8,8 +8,11 @@ import (
 	"io/ioutil"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 
+	"github.com/arangodb/docs/migration-tools/arangoproxy/internal/utils"
+	"github.com/dlclark/regexp2"
 	"gopkg.in/yaml.v3"
 )
 
@@ -89,6 +92,10 @@ func ParseExample(request io.Reader, headers http.Header) (Example, error) {
 	optionsYaml.SaveCache = headers.Get("Cache")
 	optionsYaml.Version = headers.Get("Version")
 
+	overrideRE := regexp2.MustCompile(Conf.Override, 0)
+
+	optionsYaml.SaveCache = strconv.FormatBool(utils.Regexp2StringHasMatch(overrideRE, optionsYaml.Name))
+
 	code := strings.Replace(string(decodedRequest), string(options), "", -1)
 
 	return Example{Type: "", Options: optionsYaml, Code: code, Base64Request: string(req)}, nil

toolchain/arangoproxy/internal/utils/utils.go

@@ -50,3 +50,11 @@ func Regexp2FindAllString(re *regexp2.Regexp, s string) []string {
 	}
 	return matches
 }
+
+func Regexp2StringHasMatch(re *regexp2.Regexp, s string) bool {
+	m, _ := re.FindStringMatch(s)
+	if m != nil {
+		return true
+	}
+	return false
+}

toolchain/docker/amd64/docker-compose.yml

@@ -15,6 +15,7 @@ services:
       HUGO_URL: ${HUGO_URL:-http://localhost}
       HUGO_ENV: ${HUGO_ENV:-development}
       GENERATORS: ${GENERATORS}
+      OVERRIDE: ${OVERRIDE}
       ARANGODB_SRC_3_10: ${ARANGODB_SRC_3_10}
       ARANGODB_SRC_3_11: ${ARANGODB_SRC_3_11}
       ARANGODB_SRC_3_12: ${ARANGODB_SRC_3_12}

toolchain/docker/arm64/docker-compose.yml

@@ -15,6 +15,7 @@ services:
       HUGO_URL: ${HUGO_URL:-http://localhost}
       HUGO_ENV: ${HUGO_ENV:-development}
       GENERATORS: ${GENERATORS}
+      OVERRIDE: ${OVERRIDE}
       ARANGODB_SRC_3_10: ${ARANGODB_SRC_3_10}
       ARANGODB_SRC_3_11: ${ARANGODB_SRC_3_11}
       ARANGODB_SRC_3_12: ${ARANGODB_SRC_3_12}

toolchain/scripts/start_arangoproxy.sh

@@ -19,6 +19,10 @@ ARANGOPROXY_ARGS=""
 if [ "$HUGO_ENV" != "prod" ] && [ "$HUGO_ENV" != "frontend" ]; then
   # For each server in arangoproxy/cmd/configs/local.yaml filled by previous step, check the server is up and healthy
   ARANGOPROXY_ARGS="-use-servers"
+  if [ "$OVERRIDE" != "" ] ; then
+    ARANGOPROXY_ARGS="$ARANGOPROXY_ARGS -override $OVERRIDE"
+  fi
+
   mapfile servers < <(yq e -o=j -I=0 '.repositories.[]' /home/toolchain/arangoproxy/cmd/configs/local.yaml )
 
   for server in "${servers[@]}"; do

toolchain/scripts/toolchain.sh

@@ -252,6 +252,7 @@ function run_arangoproxy_and_site() {
       -e ENV="$ENV" \
       -e HUGO_URL="$HUGO_URL" \
       -e HUGO_ENV="$HUGO_ENV" \
+      -e OVERRIDE="$OVERRIDE" \
       -v arangosh:/arangosh \
       --volumes-from toolchain \
       --log-opt tag="{{.Name}}" \
@@ -268,7 +269,7 @@ function setup_arangoproxy() {
   setup_arangoproxy_arangosh "$image" "$version"
 
   setup_arangoproxy_repositories "$version" "$container_name"
-  
+
   log "[setup_arangoproxy] Done"
 }
 
@@ -314,8 +315,6 @@ function setup_arangoproxy_repositories() {
 }
 
 
-
-
 ##### SERVER FUNCTIONS
 
 function process_server() {