Add Clouseau to the developer setup

Provide a way to the developers to deploy and launch the Clouseau
search module more easily, therefore making it more accessible.
This can help with running the Search-based Elixir tests as well as
the Mango `text` search tests.  This could allow us to catch more
bugs ahead of time and might even inspire more improvements in the
area.

The extension is designed in a way to make it simple to integrate
with the CI — only the Java environment of the proper version needs
to be deployed, everything else could be managed from this
repository.  Such as bumping the version of Clouseau or fine-tuning
the configuration parameters.

This is an opt-in feature on two levels.  First, one has to tell
`./configure --enable-clouseau` to instantiate Clouseau locally.
This will create the `clouseau` sub-directory that holds the
contents of the Clouseau distribution of the specified version,
which currently defaults to 2.22.0 (the latest).  If an older
version is needed, use the `--clouseau-version` flag.

Then the `mango-test` and `elixir-search` targets will try to use
Clouseau automatically just to make it easy to use in the old way.
However, the `dev/run` script will not do the same.  It requires
the `--with-clouseau` flag to be passed for that.  That is because
the developer may not necessarily want to launch Clouseau ad hoc,
even if it is available.

With this, note that `elixir-search` is added to the `check` target.
In lack of a configured Clouseau instance, it will become a no-op
and a warning is emitted.

Author: pgj

Makefile

@@ -157,6 +157,7 @@ check: all
 	@$(MAKE) eunit
 	@$(MAKE) mango-test
 	@$(MAKE) elixir
+	@$(MAKE) elixir-search
 	@$(MAKE) weatherreport-test
 	@$(MAKE) nouveau-test
 
@@ -263,12 +264,17 @@ elixir: elixir-init devclean
 		--no-eval 'mix test --trace --include test/elixir/test/config/suite.elixir --exclude test/elixir/test/config/skip.elixir $(EXUNIT_OPTS)'
 
 .PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau instance
+# target: elixir-search - Run search tests, requires a configured Clouseau instance
 elixir-search: export MIX_ENV=integration
 elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
 	@dev/run -n 1 -q -a adm:pass \
+		--with-clouseau \
 		--locald-config test/config/test-config.ini \
 		--no-eval 'mix test --trace --include test/elixir/test/config/search.elixir'
+else
+	@echo "Warning: Clouseau is not enabled, \`elixir-search\` cannot be run."
+endif
 
 .PHONY: elixir-source-checks
 # target: elixir-source-checks - Check source code formatting of Elixir test files
@@ -313,19 +319,24 @@ list-eunit-suites:
 build-test:
 	@test/build/test-configure.sh
 
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
 
 .PHONY: mango-test
 # target: mango-test - Run Mango tests
 mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 mango-test: devclean all
-	@cd src/mango && \
-		python3 -m venv .venv && \
-		.venv/bin/python3 -m pip install -r requirements.txt
-	@cd src/mango && \
-		../../dev/run "$(TEST_OPTS)" \
+	@python3 -m venv src/mango/.venv && \
+		src/mango/.venv/bin/python3 -m pip install -r src/mango/requirements.txt
+	@dev/run \
+		"$(TEST_OPTS)" \
+		"$(_WITH_CLOUSEAU)" \
 		-n 1 \
 		--admin=adm:pass \
-		'COUCH_USER=adm COUCH_PASS=pass .venv/bin/python3 -m nose2 $(MANGO_TEST_OPTS)'
+		--no-eval "\
+COUCH_USER=adm COUCH_PASS=pass \
+src/mango/.venv/bin/nose2 -s src/mango/test -c src/mango/unittest.cfg $(MANGO_TEST_OPTS)"
 
 
 .PHONY: weatherreport-test

Makefile.win

@@ -143,6 +143,7 @@ check: all
 	@$(MAKE) eunit
 	@$(MAKE) mango-test
 	@$(MAKE) elixir
+	@$(MAKE) elixir-search
 	@$(MAKE) nouveau-test
 
 ifdef apps
@@ -203,6 +204,8 @@ python-black-update: .venv/bin/black
 		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \
 		build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext .
 
+-include install.mk
+
 ifeq ($(with_nouveau), 0)
   exclude_nouveau=--exclude nouveau
 endif
@@ -239,12 +242,17 @@ elixir: elixir-init devclean
 		--no-eval 'mix test --trace --include test\elixir\test\config\suite.elixir --exclude test\elixir\test\config\skip.elixir $(EXUNIT_OPTS)'
 
 .PHONY: elixir-search
-# target: elixir-search - Run search tests, requires a running Clouseau instance
+# target: elixir-search - Run search tests, requires a configured Clouseau instance
 elixir-search: export MIX_ENV=integration
 elixir-search: elixir-init devclean
+ifeq ($(with_clouseau), 1)
 	@dev\run -n 1 -q -a adm:pass \
+		--with-clouseau \
 		--locald-config test/config/test-config.ini \
 		--no-eval 'mix test --trace --include test/elixir/test/config/search.elixir'
+else
+	@echo "Warning: Clouseau is not enabled, `elixir-search` cannot be run."
+endif
 
 .PHONY: elixir-source-checks
 # target: elixir-source-checks - Check source code formatting of Elixir test files
@@ -278,19 +286,24 @@ list-eunit-apps:
 list-eunit-suites:
 	@powershell -Command 'Get-ChildItem -Path src -Recurse -Filter "*_tests?.erl" | ForEach-Object { "{0}" -f $$_.BaseName } | Sort'
 
+ifeq ($(with_clouseau), 1)
+_WITH_CLOUSEAU="--with-clouseau"
+endif
 
 .PHONY: mango-test
 # target: mango-test - Run Mango tests
 mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 mango-test: devclean all
-	@cd src\mango && \
-		python.exe -m venv .venv && \
-		.venv\Scripts\pip.exe install -r requirements.txt
-	@cd src\mango && \
-		..\..\dev\run "$(TEST_OPTS)" \
+	@python.exe -m venv src\mango\.venv && \
+		src\mango\.venv\Scripts\pip.exe install -r src\mango\requirements.txt
+	@dev\run \
+		"$(TEST_OPTS)" \
+		"$(_WITH_CLOUSEAU)" \
 		-n 1 \
 		--admin=adm:pass \
-		"env COUCH_USER=adm COUCH_PASS=pass .venv\Scripts\nose2 $(MANGO_TEST_OPTS)"
+		"\
+env COUCH_USER=adm COUCH_PASS=pass \
+src\mango\.venv\Scripts\nose2 -s src\mango\test -c src\mango\unittest.cfg $(MANGO_TEST_OPTS)"
 
 
 ################################################################################
@@ -350,7 +363,6 @@ dist: all derived
 
 .PHONY: release
 # target: release - Create an Erlang release including CouchDB!
--include install.mk
 release: all
 	@echo 'Installing CouchDB into rel\couchdb\ ...'
 	-@rmdir /s/q rel\couchdb >NUL 2>&1 || true

README-DEV.rst

@@ -152,9 +152,9 @@ Configure the source by running::
 
     ./configure
 
-If you intend to run the test suites::
+If you intend to run the test suites with Clouseau::
 
-    ./configure -c
+    ./configure --enable-clouseau
 
 If you don't want to build Fauxton or documentation specify
 ``--disable-fauxton`` and/or ``--disable-docs`` arguments for ``configure`` to
@@ -237,14 +237,8 @@ but it could be done manually via the corresponding target::
 
     make elixir-search
 
-Note that this requires a running Clouseau instance with the name
-``clouseau@127.0.0.1``.  The easiest way to get it is to clone the
-`cloudant-labs/clouseau <https://github.com/cloudant-labs/clouseau>`_
-repository and launch it run there once all the prerequisites (JDK,
-Scala, and Maven) have been installed successfully, e.g.::
-
-    git clone https://github.com/cloudant-labs/clouseau
-    mvn -f clouseau/pom.xml scala:run
+Note that this requires Clouseau to be configured for running, see
+above.
 
 Mango Integration Tests
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -262,7 +256,7 @@ the implementation.  Consult its documentation for more information.
 
 Tests that rely on text indexes are run only if the ``search`` feature
 is reported to be available (i.e. a working Clouseau instance is
-connected), otherwise they will be skipped.
+configured and working), otherwise they will be skipped.
 
 Note that the databases that are created during the tests will be all
 removed after each of the suites completed.  However, with the help of
@@ -272,6 +266,51 @@ to keep those databases around for further investigation::
     MANGO_TESTS_KEEP_DBS=please \
       make mango-test MANGO_TEST_OPTS='03-operator-test'
 
+Running Clouseau
+~~~~~~~~~~~~~~~~
+
+When configured with the ``./configure`` script, the ``./dev/run``
+script is capable of launching Clouseau instances alongside the
+CouchDB nodes and hooking them up.  This is what the ``mango-test``
+and ``elixir-search`` targets also use to run their respective test
+suites, and let Clouseau automatically manage them.
+
+Although the ``./configure`` and the ``./dev/run`` scripts try to take
+care of the details of the Clouseau deployment, it is still the
+responsibility of the user to provide a suitable Java environment for
+running.  Clouseau can run with JRE 1.7 and 1.8 only.  Also, when
+Nouveau is in use, which uses a more recent Java environment, the old
+JDK has to be installed separately and the ``CLOUSEAU_JAVA_HOME``
+environment variable has to be set to point its location.
+
+Fortunately, the ```asdf`` tool <https://asdf-vm.com/>` provides a
+convenient way to install old versions of JDK through its ```java``
+plugin <https://github.com/halcyon/asdf-java>`::
+
+    asdf plugin add java
+
+Then use ``asdf`` to install it::
+
+    asdf install java zulu-jre-8.74.0.17
+
+Finally, use ``asdf`` to set the ``CLOUSEAU_JAVA_HOME`` environment
+variable::
+
+    export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-jre-8.74.0.17)
+
+If the use of ``asdf`` is not an option, `the Zulu site
+<https://cdn.azul.com/zulu/bin/>` could be used directly to get the
+distribution package for the appropriate JRE version.  But this is
+just one of the possibilities to access installers for old Java
+environments.
+
+Once both Clouseau and the corresponding Java environment are set,
+they are not put in use automatically.  In order to do so, the
+``./dev/run`` script needs to be run with Clouseau enabled as
+follows::
+
+    dev/run --with-clouseau
+
 Static Code Analysis
 ~~~~~~~~~~~~~~~~~~~~
 

configure

@@ -29,6 +29,7 @@ WITH_PROPER="true"
 WITH_FAUXTON=1
 WITH_DOCS=1
 WITH_NOUVEAU=0
+WITH_CLOUSEAU=0
 ERLANG_MD5="false"
 SKIP_DEPS=0
 
@@ -38,6 +39,8 @@ run_erlang() {
 
 COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)"
 SM_VSN=${SM_VSN:-"91"}
+CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.22.0"}
+CLOUSEAU_DIR="$(pwd)"/clouseau
 ARCH="$(uname -m)"
 ERLANG_VER="$(run_erlang 'io:put_chars(erlang:system_info(otp_release)).')"
 ERLANG_OS="$(run_erlang 'case os:type() of {OS, _} -> io:format("~s~n", [OS]) end.')"
@@ -59,9 +62,11 @@ Options:
   --disable-fauxton           do not build Fauxton
   --disable-docs              do not build any documentation or manpages
   --enable-nouveau            enable the new experimental search module
+  --enable-clouseau           enable the Clouseau search module
   --erlang-md5                use erlang for md5 hash operations
   --dev                       alias for --disable-docs --disable-fauxton
   --spidermonkey-version VSN  specify the version of SpiderMonkey to use (defaults to $SM_VSN)
+  --clouseau-version VSN      specify the version of Clouseau to use (defaults to $CLOUSEAU_VSN)
   --skip-deps                 do not update erlang dependencies
   --rebar=PATH                use rebar by specified path (version >=2.6.0 && <3.0 required)
   --rebar3=PATH               use rebar3 by specified path
@@ -101,6 +106,12 @@ parse_opts() {
                 continue
                 ;;
 
+	    --enable-clouseau)
+		WITH_CLOUSEAU=1
+		shift
+		continue
+		;;
+
             --erlang-md5)
                 ERLANG_MD5="true"
                 shift
@@ -202,6 +213,24 @@ parse_opts() {
                 exit 1
                 ;;
 
+	    --clouseau-version)
+		if [ -n "$2" ]; then
+		    eval CLOUSEAU_SVN=$2
+		    shift 2
+		    continue
+		else
+		    printf 'ERROR: "--clouseau-version" requires a non-empty argument.\n' >&2
+		    exit 1
+		fi
+		;;
+	    --clouseau-version=?*)
+		eval CLOUSEAU_VSN=${1#*=}
+		;;
+	    --clouseau-version=)
+	        printf 'ERROR: "--clouseau-version" requires a non-empty argument.\n' >&2
+	        exit 1
+	        ;;
+
             --) # End of options
                 shift
                 break
@@ -298,6 +327,7 @@ package_author_name = $PACKAGE_AUTHOR_NAME
 with_fauxton = $WITH_FAUXTON
 with_docs = $WITH_DOCS
 with_nouveau = $WITH_NOUVEAU
+with_clouseau = $WITH_CLOUSEAU
 
 user = $COUCHDB_USER
 spidermonkey_version = $SM_VSN
@@ -345,6 +375,45 @@ install_local_erlfmt() {
     fi
 }
 
+install_local_clouseau() {
+    _DIST_URL=https://github.com/cloudant-labs/clouseau/releases/download/"$CLOUSEAU_VSN"/clouseau-"$CLOUSEAU_VSN"-dist.zip
+    _MAVEN_BASE_URI=https://repo1.maven.org/maven2
+
+    _SLF4J_SIMPLE_VSN=${SLF4J_SIMPLE_VERSION:-1.7.36}
+    _SLF4J_SIMPLE_JAR=slf4j-simple-"$_SLF4J_SIMPLE_VSN".jar
+    _SLF4J_SIMPLE_URL="$_MAVEN_BASE_URI"/org/slf4j/slf4j-simple/"$_SLF4J_SIMPLE_VSN"/"$_SLF4J_SIMPLE_JAR"
+
+    rm -rf "$CLOUSEAU_DIR"
+    mkdir -p "$CLOUSEAU_DIR"
+
+    if ! curl -sSL --max-redirs 1 -o clouseau.zip "$_DIST_URL"; then
+	printf "ERROR: %s could not be downloaded.\n" "$_DIST_URL" >&2
+	exit 1
+    fi
+
+    if ! unzip -q -j clouseau.zip -d "$CLOUSEAU_DIR"; then
+	printf "ERROR: Clouseau distribution package (clouseau.zip) could not be extracted.\n" >&2
+	exit 1
+    fi
+
+    rm clouseau.zip
+
+    if ! curl -sSL --max-redirs 1 -o "$CLOUSEAU_DIR"/"$_SLF4J_SIMPLE_JAR" "$_SLF4J_SIMPLE_URL"; then
+	printf "ERROR: %s could not be downloaded.\n" "$_SLF4J_SIMPLE_URL" >&2
+	exit 1
+    fi
+
+    cat <<EOF > "$CLOUSEAU_DIR"/clouseau.ini
+[clouseau]
+EOF
+    cat <<EOF > "$CLOUSEAU_DIR"/log4j.properties
+log4j.rootLogger=debug, CONSOLE
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n
+EOF
+}
+
 if [ -z "${REBAR}" ]; then
     install_local_rebar
     REBAR=${rootdir}/bin/rebar
@@ -360,6 +429,12 @@ if [ -z "${ERLFMT}" ]; then
     ERLFMT=${rootdir}/bin/erlfmt
 fi
 
+if [ $WITH_CLOUSEAU -ne 0 ]; then
+    install_local_clouseau
+else
+    rm -rf "$CLOUSEAU_DIR"
+fi
+
 # only update dependencies, when we are not in a release tarball
 if [ -d .git  -a $SKIP_DEPS -ne 1 ]; then
     echo "==> updating dependencies"