Add Clouseau to the developer setup

Provide a way to the developers to deploy and launch the Clouseau
search module more easily, therefore making it more accessible.
This can help with running the Search-based Elixir tests as well as
the Mango `text` search tests.  This could allow us to catch more
bugs ahead of time and might even inspire more improvements in the
area.

The extension is designed in a way to make it simple to integrate
with the CI — only the Java environment of the proper version needs
to be deployed, everything else could be managed from this
repository.  Such as bumping the version of Clouseau or fine-tuning
the configuration parameters.

This is an opt-in feature on two levels.  First, one has to tell
`./configure --enable-clouseau` to instantiate Clouseau locally.
This will create the `clouseau` sub-directory that holds the
contents of the Clouseau distribution of the specified version,
which currently defaults to 2.22.0 (the latest).  If an older
version is needed, use the `--clouseau-version` flag.

Then the `mango-test` and `elixir-search` targets will try to use
Clouseau automatically just to make it easy to use in the old way.
However, the `dev/run` script will not do the same.  It requires
the `--with-clouseau` flag to be passed for that.  That is because
the developer may not necessarily want to launch Clouseau ad hoc,
even if it is available.

With this, note that `elixir-search` is added to the `check` target.
In lack of a configured Clouseau instance, it will become a no-op
and a warning is emitted.

Author: pgj

.devcontainer/devcontainer.json

@@ -5,11 +5,11 @@
         "context": "..",
         "args": {
             // Useful choices include:
-            // apache/couchdbci-debian:bullseye-erlang-25.0.2
-            // apache/couchdbci-debian:bullseye-erlang-24.3.4.2
-            // apache/couchdbci-debian:bullseye-erlang-23.3.4.15
+            // apache/couchdbci-debian:bullseye-erlang-26.1.2
+            // apache/couchdbci-debian:bullseye-erlang-25.3.2.7
+            // apache/couchdbci-debian:bookworm-erlang-24.3.4.14
             //
-            "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-24.3.4.10"
+            "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-24.3.4.14"
         }
     },
 
@@ -23,7 +23,7 @@
     // Your code will live inside the volume created for the container under /workspace.
     "workspaceMount": "target=/workspaces/couchdb,type=volume",
     "workspaceFolder": "/workspaces/couchdb",
-    "postCreateCommand": "./configure --enable-nouveau && make",
+    "postCreateCommand": "./configure --enable-nouveau --enable-clouseau && make",
 
     "customizations": {
         "vscode": {

Makefile

@@ -157,6 +157,7 @@ check: all
 	@$(MAKE) eunit
 	@$(MAKE) mango-test
 	@$(MAKE) elixir
+	@$(MAKE) elixir-search
 	@$(MAKE) weatherreport-test
 	@$(MAKE) nouveau-test
 
@@ -263,12 +264,17 @@ elixir: elixir-init devclean
 		--no-eval 'mix test --trace --include test/elixir/test/config/suite.elixir --exclude test/elixir/test/config/skip.elixir $(EXUNIT_OPTS)'
 
 .PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau instance
+# target: elixir-search - Run search tests, requires a configured Clouseau instance
 elixir-search: export MIX_ENV=integration
 elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
 	@dev/run -n 1 -q -a adm:pass \
+		--with-clouseau \
 		--locald-config test/config/test-config.ini \
 		--no-eval 'mix test --trace --include test/elixir/test/config/search.elixir'
+else
+	@echo "Warning: Clouseau is not enabled, \`elixir-search\` cannot be run."
+endif
 
 .PHONY: elixir-source-checks
 # target: elixir-source-checks - Check source code formatting of Elixir test files
@@ -313,19 +319,24 @@ list-eunit-suites:
 build-test:
 	@test/build/test-configure.sh
 
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
 
 .PHONY: mango-test
 # target: mango-test - Run Mango tests
 mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 mango-test: devclean all
-	@cd src/mango && \
-		python3 -m venv .venv && \
-		.venv/bin/python3 -m pip install -r requirements.txt
-	@cd src/mango && \
-		../../dev/run "$(TEST_OPTS)" \
+	@python3 -m venv src/mango/.venv && \
+		src/mango/.venv/bin/python3 -m pip install -r src/mango/requirements.txt
+	@dev/run \
+		"$(TEST_OPTS)" \
+		"$(_WITH_CLOUSEAU)" \
 		-n 1 \
 		--admin=adm:pass \
-		'COUCH_USER=adm COUCH_PASS=pass .venv/bin/python3 -m nose2 $(MANGO_TEST_OPTS)'
+		--no-eval "\
+COUCH_USER=adm COUCH_PASS=pass \
+src/mango/.venv/bin/nose2 -s src/mango/test -c src/mango/unittest.cfg $(MANGO_TEST_OPTS)"
 
 
 .PHONY: weatherreport-test

Makefile.win

@@ -143,6 +143,7 @@ check: all
 	@$(MAKE) eunit
 	@$(MAKE) mango-test
 	@$(MAKE) elixir
+	@$(MAKE) elixir-search
 	@$(MAKE) nouveau-test
 
 ifdef apps
@@ -203,6 +204,8 @@ python-black-update: .venv/bin/black
 		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \
 		build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext .
 
+-include install.mk
+
 ifeq ($(with_nouveau), 0)
   exclude_nouveau=--exclude nouveau
 endif
@@ -239,12 +242,17 @@ elixir: elixir-init devclean
 		--no-eval 'mix test --trace --include test\elixir\test\config\suite.elixir --exclude test\elixir\test\config\skip.elixir $(EXUNIT_OPTS)'
 
 .PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau instance
+# target: elixir-search - Run search tests, requires a configured Clouseau instance
 elixir-search: export MIX_ENV=integration
 elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
 	@dev\run -n 1 -q -a adm:pass \
+		--with-clouseau \
 		--locald-config test/config/test-config.ini \
 		--no-eval 'mix test --trace --include test/elixir/test/config/search.elixir'
+else
+	@echo "Warning: Clouseau is not enabled, `elixir-search` cannot be run."
+endif
 
 .PHONY: elixir-source-checks
 # target: elixir-source-checks - Check source code formatting of Elixir test files
@@ -278,19 +286,24 @@ list-eunit-apps:
 list-eunit-suites:
 	@powershell -Command 'Get-ChildItem -Path src -Recurse -Filter "*_tests?.erl" | ForEach-Object { "{0}" -f $$_.BaseName } | Sort'
 
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
 
 .PHONY: mango-test
 # target: mango-test - Run Mango tests
 mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 mango-test: devclean all
-	@cd src\mango && \
-		python.exe -m venv .venv && \
-		.venv\Scripts\pip.exe install -r requirements.txt
-	@cd src\mango && \
-		..\..\dev\run "$(TEST_OPTS)" \
+	@$(PYTHON) -m venv src\mango\.venv && \
+		src\mango\.venv\Scripts\pip.exe install -r src\mango\requirements.txt
+	@dev\run \
+		"$(TEST_OPTS)" \
+		"$(_WITH_CLOUSEAU)" \
 		-n 1 \
 		--admin=adm:pass \
-		"env COUCH_USER=adm COUCH_PASS=pass .venv\Scripts\nose2 $(MANGO_TEST_OPTS)"
+		"\
+env COUCH_USER=adm COUCH_PASS=pass \
+src\mango\.venv\Scripts\nose2 -s src\mango\test -c src\mango\unittest.cfg $(MANGO_TEST_OPTS)"
 
 
 ################################################################################
@@ -350,7 +363,6 @@ dist: all derived
 
 .PHONY: release
 # target: release - Create an Erlang release including CouchDB!
--include install.mk
 release: all
 	@echo 'Installing CouchDB into rel\couchdb\ ...'
 	-@rmdir /s/q rel\couchdb >NUL 2>&1 || true

README-DEV.rst

@@ -152,9 +152,9 @@ Configure the source by running::
 
     ./configure
 
-If you intend to run the test suites::
+If you intend to run the test suites with Clouseau::
 
-    ./configure -c
+    ./configure --enable-clouseau
 
 If you don't want to build Fauxton or documentation specify
 ``--disable-fauxton`` and/or ``--disable-docs`` arguments for ``configure`` to
@@ -237,14 +237,8 @@ but it could be done manually via the corresponding target::
 
     make elixir-search
 
-Note that this requires a running Clouseau instance with the name
-``clouseau@127.0.0.1``.  The easiest way to get it is to clone the
-`cloudant-labs/clouseau <https://github.com/cloudant-labs/clouseau>`_
-repository and launch it run there once all the prerequisites (JDK,
-Scala, and Maven) have been installed successfully, e.g.::
-
-    git clone https://github.com/cloudant-labs/clouseau
-    mvn -f clouseau/pom.xml scala:run
+Note that this requires Clouseau to be configured for running, see
+above.
 
 Mango Integration Tests
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -262,7 +256,7 @@ the implementation.  Consult its documentation for more information.
 
 Tests that rely on text indexes are run only if the ``search`` feature
 is reported to be available (i.e. a working Clouseau instance is
-connected), otherwise they will be skipped.
+configured and working), otherwise they will be skipped.
 
 Note that the databases that are created during the tests will be all
 removed after each of the suites completed.  However, with the help of
@@ -272,6 +266,58 @@ to keep those databases around for further investigation::
     MANGO_TESTS_KEEP_DBS=please \
       make mango-test MANGO_TEST_OPTS='03-operator-test'
 
+Running Clouseau
+~~~~~~~~~~~~~~~~
+
+When configured with the ``./configure`` script, the ``./dev/run``
+script is capable of launching Clouseau instances alongside the
+CouchDB nodes and hooking them up.  This is what the ``mango-test``
+and ``elixir-search`` targets also use to run their respective test
+suites, and let Clouseau automatically manage them.
+
+Although the ``./configure`` and the ``./dev/run`` scripts try to take
+care of the details of the Clouseau deployment, it is still the
+responsibility of the user to provide a suitable Java environment for
+running.  Clouseau can run with JRE 1.7 and 1.8 only.  Also, when
+Nouveau is in use, which uses a more recent Java environment, the old
+JDK has to be installed separately and the ``CLOUSEAU_JAVA_HOME``
+environment variable has to be set to point its location.
+
+Fortunately, the ```asdf`` tool <https://asdf-vm.com/>` provides a
+convenient way to install old versions of JDK through its ```java``
+plugin <https://github.com/halcyon/asdf-java>`::
+
+    asdf plugin add java
+
+Then use ``asdf`` to install it::
+
+    asdf install java zulu-jre-8.74.0.17
+
+Finally, use ``asdf`` to set the ``CLOUSEAU_JAVA_HOME`` environment
+variable::
+
+    export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-jre-8.74.0.17)
+
+If the use of ``asdf`` is not an option, `the Zulu site
+<https://cdn.azul.com/zulu/bin/>` could be used directly to get the
+distribution package for the appropriate JRE version.  But this is
+just one of the possibilities to access installers for old Java
+environments.
+
+Once both Clouseau and the corresponding Java environment are set,
+they are not put in use automatically.  In order to do so, the
+``./dev/run`` script needs to be run with Clouseau enabled as
+follows::
+
+    dev/run --with-clouseau
+
+When a specific Erlang cookie string is set in
+``rel/overlay/etc/vm.args``, the ``--erlang-cookie`` flag could be
+used to configure Clouseau to work with that::
+
+    dev/run --with-clouseau --erlang-cookie=brumbrum
+
+
 Static Code Analysis
 ~~~~~~~~~~~~~~~~~~~~
 

build-aux/Jenkinsfile.full

@@ -27,41 +27,47 @@ meta = [
     name: 'CentOS 7',
     spidermonkey_vsn: '1.8.5',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
   ],
 
   'centos8': [
     name: 'CentOS 8',
     spidermonkey_vsn: '60',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
   ],
 
   'bionic': [
     name: 'Ubuntu 18.04',
     spidermonkey_vsn: '1.8.5',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
   ],
 
   'focal': [
     name: 'Ubuntu 20.04',
     spidermonkey_vsn: '68',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
   ],
 
   'jammy': [
     name: 'Ubuntu 22.04',
     spidermonkey_vsn: '91',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}"
   ],
 
   'buster': [
     name: 'Debian 10',
     spidermonkey_vsn: '60',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
   ],
 
@@ -72,6 +78,7 @@ meta = [
   //   name: 'Debian 11 ARM',
   //   spidermonkey_vsn: '78',
   //   enable_nouveau: true,
+  //   enable_clouseau: false,
   //   image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}",
   //   node_label: 'arm64v8'
   // ],
@@ -80,6 +87,7 @@ meta = [
     name: 'Debian 11 POWER',
     spidermonkey_vsn: '78',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}",
     node_label: 'ppc64le'
   ],
@@ -98,32 +106,37 @@ meta = [
     name: 'Debian 11',
     spidermonkey_vsn: '78',
     enable_nouveau: true,
+    enable_clouseau: true,
     image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
   ],
 
   'bookworm': [
     name: 'Debian 12',
     spidermonkey_vsn: '78',
     enable_nouveau: true,
+    enable_clouseau: false,
     image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}"
   ],
 
   'freebsd-x86_64': [
      name: 'FreeBSD',
      spidermonkey_vsn: '91',
+     enable_clouseau: false,
      gnu_make: 'gmake'
   ],
 
   'freebsd-arm64': [
      name: 'FreeBSD',
      spidermonkey_vsn: '91',
+     enable_clouseau: false,
      gnu_make: 'gmake'
   ],
 
  'macos': [
     name: 'macOS',
     spidermonkey_vsn: '91',
     enable_nouveau: false,
+    enable_clouseau: false,
     gnu_make: 'make'
   ]
 ]
@@ -133,6 +146,9 @@ def String configure(config) {
   if (config.enable_nouveau) {
     result += " --enable-nouveau"
   }
+  if (config.enable_clouseau) {
+    result += " --enable-clouseau"
+  }
   return result
 }
 
@@ -170,6 +186,7 @@ def generateNativeStage(platform) {
                 sh '$MAKE'
                 sh '$MAKE eunit'
                 sh '$MAKE elixir'
+                sh '$MAKE elixir-search'
                 sh '$MAKE mango-test'
                 sh '$MAKE weatherreport-test'
                 sh '$MAKE nouveau-test'
@@ -223,6 +240,7 @@ def generateContainerStage(platform) {
                     sh 'make'
                     sh 'make eunit'
                     sh 'make elixir'
+                    sh 'make elixir-search'
                     sh 'make mango-test'
                     sh 'make weatherreport-test'
                     sh 'make nouveau-test'