Use Erlang.mk's native Elixir support for CLI (#13613)

This avoids using Mix while compiling which simplifies
a number of things and let us do further build improvements
later on.

Elixir is only enabled from within rabbitmq_cli currently.

Eunit is disabled since there are only Elixir tests.

Dialyzer will force-enable Elixir in order to process
Elixir-compiled beam files.

This commit also includes a few changes that are
related:

 * The Erlang distribution will now be started for parallel-ct

 * Many unnecessary PROJECT_MOD lines have been removed

 * `eunit_formatters` has been removed, it provides little value

 * The new `maybe_flock` Erlang.mk function is used where possible

 * Build test deps when testing rabbitmq_cli (Mix won't do it anymore)

 * rabbitmq_ct_helpers now use the early plugins to have Dialyzer
   properly set up

(cherry picked from commit c5d150a)

Co-authored-by: Loïc Hoguin <loic.hoguin@broadcom.com>

Author: michaelklishin

Makefile

@@ -31,10 +31,6 @@ DISABLE_DISTCLEAN = 1
 ifeq ($(filter-out xref,$(MAKECMDGOALS)),)
 XREF_SCOPE = app deps
 
-# We add all the applications that are in non-standard paths
-# so they are included in the analyses as well.
-XREF_EXTRA_APP_DIRS = $(filter-out deps/rabbitmq_cli/_build/dev/lib/rabbit_common/,$(wildcard deps/rabbitmq_cli/_build/dev/lib/*/)) deps/rabbitmq_prelaunch/
-
 # For Elixir protocols the right fix is to include the consolidated/
 # folders in addition to ebin/. However this creates conflicts because
 # some modules are duplicated. So instead we ignore warnings from
@@ -49,10 +45,6 @@ XREF_IGNORE = [ \
 xref: ERL_LIBS := $(ERL_LIBS):$(CURDIR)/apps:$(CURDIR)/deps:$(dir $(shell elixir --eval ':io.format "~s~n", [:code.lib_dir :elixir ]'))
 endif
 
-ifneq ($(wildcard deps/.hex/cache.erl),)
-deps:: restore-hex-cache-ets-file
-endif
-
 include rabbitmq-components.mk
 
 # Set PROJECT_VERSION, calculated in rabbitmq-components.mk,
@@ -84,54 +76,6 @@ ifdef PLUGINS
 RABBITMQ_ENABLED_PLUGINS ?= $(call comma_list,$(PLUGINS))
 endif
 
-# --------------------------------------------------------------------
-# Mix Hex cache management.
-# --------------------------------------------------------------------
-
-# We restore the initial Hex cache.ets file from an Erlang term created
-# at the time the source archive was prepared.
-#
-# See the `$(SOURCE_DIST)` recipe for the reason behind this step.
-
-restore-hex-cache-ets-file: deps/.hex/cache.ets
-
-deps/.hex/cache.ets: deps/.hex/cache.erl
-	$(gen_verbose) $(call erlang,$(call restore_hex_cache_from_erl_term,$<,$@))
-
-define restore_hex_cache_from_erl_term
-  In = "$(1)",
-  Out = "$(2)",
-  {ok, [Props, Entries]} = file:consult(In),
-  Name = proplists:get_value(name, Props),
-  Type = proplists:get_value(type, Props),
-  Access = proplists:get_value(protection, Props),
-  NamedTable = proplists:get_bool(named_table, Props),
-  Keypos = proplists:get_value(keypos, Props),
-  Heir = proplists:get_value(heir, Props),
-  ReadConc = proplists:get_bool(read_concurrency, Props),
-  WriteConc = proplists:get_bool(write_concurrency, Props),
-  Compressed = proplists:get_bool(compressed, Props),
-  Options0 = [
-    Type,
-    Access,
-    {keypos, Keypos},
-    {heir, Heir},
-    {read_concurrency, ReadConc},
-    {write_concurrency, WriteConc}],
-  Options1 = case NamedTable of
-    true  -> [named_table | Options0];
-    false -> Options0
-  end,
-  Options2 = case Compressed of
-    true  -> [compressed | Options0];
-    false -> Options0
-  end,
-  Tab = ets:new(Name, Options2),
-  [true = ets:insert(Tab, Entry) || Entry <- Entries],
-  ok = ets:tab2file(Tab, Out),
-  init:stop().
-endef
-
 # --------------------------------------------------------------------
 # Distribution - common variables and generic functions.
 # --------------------------------------------------------------------
@@ -263,14 +207,6 @@ $(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
 		sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$$$|include ../../erlang.mk|" \
 		 $$@/deps/$$$$(basename $$$$dep)/Makefile && \
 		rm $$@/deps/$$$$(basename $$$$dep)/Makefile.bak; \
-		mix_exs=$$@/deps/$$$$(basename $$$$dep)/mix.exs; \
-		if test -f $$$$mix_exs; then \
-			(cd $$$$(dirname "$$$$mix_exs") && \
-			 (test -d $$@/deps/.hex || env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
-			 env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
-			 cp $(CURDIR)/mk/rabbitmq-mix.mk . && \
-			 rm -rf _build deps); \
-		fi; \
 		if test -f "$$$$dep/license_info"; then \
 			cp "$$$$dep/license_info" "$$@/deps/licensing/license_info_$$$$(basename $$$$dep)"; \
 			cat "$$$$dep/license_info" >> $$@/LICENSE; \
@@ -295,7 +231,6 @@ $(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
 	done
 	$${verbose} echo "PLUGINS := $(PLUGINS)" > $$@/plugins.mk
 	$${verbose} sort -r < "$$@.git-times.txt" | head -n 1 > "$$@.git-time.txt"
-	$${verbose} $$(call erlang,$$(call dump_hex_cache_to_erl_term,$$(call core_native_path,$$@),$$(call core_native_path,$$@.git-time.txt)))
 	$${verbose} find $$@ -print0 | xargs -0 touch -t "$$$$(cat $$@.git-time.txt)"
 	$${verbose} rm "$$@.git-times.txt" "$$@.git-time.txt"
 
@@ -337,47 +272,6 @@ clean-$(1):
 clean:: clean-$(1)
 endef
 
-# Mix Hex component requires a cache file, otherwise it refuses to build
-# offline... That cache is an ETS table with all the applications we
-# depend on, plus some versioning informations and checksums. There
-# are two problems with that: the table contains a date (`last_update`
-# field) and `ets:tab2file()` produces a different file each time it's
-# called.
-#
-# To make our source archive reproducible, we fix the time of the
-# `last_update` field to the last Git commit and dump the content of the
-# table as an Erlang term to a text file.
-#
-# The ETS file must be recreated before compiling RabbitMQ. See the
-# `restore-hex-cache-ets-file` Make target.
-define dump_hex_cache_to_erl_term
-  In = "$(1)/deps/.hex/cache.ets",
-  Out = "$(1)/deps/.hex/cache.erl",
-  {ok, DateStr} = file:read_file("$(2)"),
-  {match, Date} = re:run(DateStr,
-    "^([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})\.([0-9]{2})",
-    [{capture, all_but_first, list}]),
-  [Year, Month, Day, Hour, Min, Sec] = [erlang:list_to_integer(V) || V <- Date],
-  {ok, Tab} = ets:file2tab(In),
-  true = ets:insert(Tab, {last_update, {{Year, Month, Day}, {Hour, Min, Sec}}}),
-  Props = [
-    Prop
-    || {Key, _} = Prop <- ets:info(Tab),
-    Key =:= name orelse
-    Key =:= type orelse
-    Key =:= protection orelse
-    Key =:= named_table orelse
-    Key =:= keypos orelse
-    Key =:= heir orelse
-    Key =:= read_concurrency orelse
-    Key =:= write_concurrency orelse
-    Key =:= compressed],
-  Entries = ets:tab2list(Tab),
-  ok = file:write_file(Out, io_lib:format("~w.~n~w.~n", [Props, Entries])),
-  ok = file:delete(In),
-  init:stop().
-endef
-
 # --------------------------------------------------------------------
 # Distribution - public targets
 # --------------------------------------------------------------------

deps/amqp10_client/Makefile

@@ -1,6 +1,5 @@
 PROJECT = amqp10_client
 PROJECT_DESCRIPTION = AMQP 1.0 client
-PROJECT_MOD = amqp10_client_app
 
 define PROJECT_APP_EXTRA_KEYS
 %% Hex.pm package informations.

deps/oauth2_client/Makefile

@@ -1,6 +1,5 @@
 PROJECT = oauth2_client
 PROJECT_DESCRIPTION = OAuth2 client from the RabbitMQ Project
-PROJECT_MOD = oauth2_client_app
 
 BUILD_DEPS = rabbit
 DEPS = rabbit_common jose
@@ -12,5 +11,8 @@ PLT_APPS = rabbit
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk
 
+# Required to properly autopatch jose.
+ELIXIR = system
+
 include rabbitmq-components.mk
 include erlang.mk

deps/rabbit/Makefile

@@ -328,6 +328,7 @@ tpl_parallel_ct_test_spec_set_$1 = $$(call tpl_parallel_ct_test_spec,$(PARALLEL_
 parallel-ct-set-$(1): test-build
 	$(verbose) mkdir -p $(CT_LOGS_DIR)
 	$(verbose) $$(call core_render,tpl_parallel_ct_test_spec_set_$(1),ct.set-$(1).spec)
+	$$(eval ERL := erl -noinput -boot no_dot_erlang)
 	$$(call erlang,$$(call ct_master.erl,ct.set-$(1).spec),-sname parallel_ct_$(PROJECT)@localhost -hidden -kernel net_ticktime 5)
 endef
 
@@ -337,6 +338,7 @@ $(foreach set,1 2 3 4,$(eval $(call parallel_ct_set_target,$(set))))
 
 parallel-ct: test-build
 	$(verbose) mkdir -p $(CT_LOGS_DIR)
+	$(eval ERL := erl -noinput -boot no_dot_erlang)
 	$(call erlang,$(call ct_master.erl,ct.test.spec),-sname parallel_ct_$(PROJECT)@localhost -hidden -kernel net_ticktime 5)
 
 # --------------------------------------------------------------------

deps/rabbit/src/rabbit_variable_queue.erl

@@ -1880,6 +1880,7 @@ determine_persist_to(Msg,
     %% via the direct client), we make a guess based on the number of
     %% headers.
 
+    %% @todo We can probably simplify this.
     {MetaSize, _BodySize} = mc:size(Msg),
      case BodySize >= IndexMaxSize of
          true  -> msg_store;

deps/rabbit_common/mk/rabbitmq-build.mk

@@ -12,7 +12,7 @@ ifneq ($(filter rabbitmq_cli,$(BUILD_DEPS) $(DEPS)),)
 # Add the CLI ebin directory to the code path for the compiler: plugin
 # CLI extensions may access behaviour modules defined in this directory.
 
-RMQ_ERLC_OPTS += -pa $(DEPS_DIR)/rabbitmq_cli/_build/dev/lib/rabbitmqctl/ebin
+RMQ_ERLC_OPTS += -pa $(DEPS_DIR)/rabbitmq_cli/ebin
 endif
 
 RMQ_ERLC_OPTS += +deterministic

deps/rabbit_common/mk/rabbitmq-dist.mk

@@ -3,7 +3,6 @@
 DIST_DIR ?= $(CURDIR)/plugins
 CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
 CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
-MIX = echo y | mix
 
 # Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of
 # plugins as .ez archives.
@@ -81,17 +80,13 @@ endef
 
 # Real entry point: it tests the existence of an .app file to determine
 # if it is an Erlang application (and therefore if it should be provided
-# as an .ez plugin archive) and calls do_ez_target_erlangmk. If instead
-# it finds a Mix configuration file, it is skipped, as the only elixir
-# applications in the directory are used by rabbitmq_cli and compiled
-# with it.
+# as an .ez plugin archive) and calls do_ez_target_erlangmk.
 #
 #   $(call ez_target,path_to_app)
 
 define ez_target
 dist_$(1)_appdir  = $(2)
 dist_$(1)_appfile = $$(dist_$(1)_appdir)/ebin/$(1).app
-dist_$(1)_mixfile = $$(dist_$(1)_appdir)/mix.exs
 
 $$(if $$(shell test -f $$(dist_$(1)_appfile) && echo OK), \
   $$(eval $$(call do_ez_target_erlangmk,$(1),$$(call get_app_version,$$(dist_$(1)_appfile)),$$(dist_$(1)_appdir))))
@@ -117,9 +112,8 @@ endif
 endif
 
 # The actual recipe to create the .ez plugin archive. Some variables
-# are defined in the do_ez_target_erlangmk and do_ez_target_mix macros
-# above. All .ez archives are also listed in this do_ez_target_erlangmk
-# and do_ez_target_mix macros.
+# are defined in the do_ez_target_erlangmk macro
+# above. All .ez archives are also listed in this macro.
 
 RSYNC ?= rsync
 RSYNC_V_0 =
@@ -200,7 +194,7 @@ test-dist:: $(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST) test-build
 		$(MAYBE_APPS_LIST)"; \
 	fi
 
-DIST_EZS = $(ERLANGMK_DIST_EZS) $(MIX_DIST_EZS)
+DIST_EZS = $(ERLANGMK_DIST_EZS)
 
 do-dist:: $(DIST_EZS)
 	$(verbose) unwanted='$(filter-out $(DIST_EZS) $(EXTRA_DIST_EZS), \
@@ -223,43 +217,21 @@ endif
 install-cli: install-cli-scripts install-cli-escripts
 	@:
 
-install-cli-scripts:
+install-cli-scripts: | $(CLI_SCRIPTS_DIR)
 	$(gen_verbose) \
 	set -e; \
 	test -d "$(DEPS_DIR)/rabbit/scripts"; \
-	if command -v flock >/dev/null; then \
-		flock $(CLI_SCRIPTS_LOCK) \
-		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/'; \
-	elif command -v lockf >/dev/null; then \
-		lockf $(CLI_SCRIPTS_LOCK) \
-		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/'; \
-	else \
-		mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/; \
-	fi
+	$(call maybe_flock,$(CLI_SCRIPTS_LOCK), \
+		cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/)
 
-install-cli-escripts:
-	$(gen_verbose) \
-	if command -v flock >/dev/null; then \
-		flock $(CLI_ESCRIPTS_LOCK) \
-		sh -c 'mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
-		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
-			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
-			DESTDIR='; \
-	elif command -v lockf >/dev/null; then \
-		lockf $(CLI_ESCRIPTS_LOCK) \
-		sh -c 'mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
+install-cli-escripts: | $(CLI_ESCRIPTS_DIR)
+	$(gen_verbose) $(call maybe_flock,$(CLI_ESCRIPTS_LOCK), \
 		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
 			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
-			DESTDIR='; \
-	else \
-		mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
-		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
-			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
-			DESTDIR= ; \
-	fi
+			DESTDIR= IS_DEP=1)
+
+$(CLI_SCRIPTS_DIR) $(CLI_ESCRIPTS_DIR):
+	$(verbose) mkdir -p $@
 
 clean-dist::
 	$(gen_verbose) rm -rf \

deps/rabbit_common/mk/rabbitmq-early-plugin.mk

@@ -4,7 +4,8 @@
 
 DIALYZER_OPTS ?= -Werror_handling -Wunmatched_returns -Wunknown
 
-dialyze: ERL_LIBS = $(APPS_DIR):$(DEPS_DIR):$(DEPS_DIR)/rabbitmq_cli/_build/dev/lib:$(dir $(shell elixir --eval ':io.format "~s~n", [:code.lib_dir :elixir ]'))
+dialyze: ELIXIR_LIBS = $(dir $(shell readlink -f `which elixir`))/../lib
+dialyze: ERL_LIBS = $(APPS_DIR):$(DEPS_DIR):$(ELIXIR_LIBS)
 
 # --------------------------------------------------------------------
 # Common Test flags.

deps/rabbitmq_auth_backend_cache/Makefile

@@ -19,7 +19,7 @@ endef
 DEPS = rabbit_common rabbit
 TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers
 
-PLT_APPS += rabbitmqctl
+PLT_APPS += rabbitmq_cli
 
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

deps/rabbitmq_auth_backend_oauth2/Makefile

@@ -10,7 +10,7 @@ BUILD_DEPS = rabbit_common rabbitmq_cli
 DEPS = rabbit cowlib jose base64url oauth2_client
 TEST_DEPS = cowboy rabbitmq_web_dispatch rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client rabbitmq_web_mqtt emqtt rabbitmq_amqp_client
 
-PLT_APPS += rabbitmqctl
+PLT_APPS += rabbitmq_cli
 
 DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk
@@ -19,5 +19,8 @@ dep_base64url = hex 1.0.1
 
 dep_emqtt = git https://github.com/emqx/emqtt.git 1.11.0
 
+# Required to properly autopatch jose.
+ELIXIR = system
+
 include ../../rabbitmq-components.mk
 include ../../erlang.mk