Refactoring docker-compose.yml & tester & debugger

README-ja.md

@@ -16,6 +16,7 @@ QuickMPCはPEC（Privacy-Enhancing Computation）のひとつであるSMPC（Sec
 - `make`
 - `docker`
 - `docker-compose`
+- [`vscode-yaml-sort`](https://marketplace.visualstudio.com/items?itemName=PascalReitermann93.vscode-yaml-sort)
 
 ## Getting Started
 [QuickMPC-libClient-pyのデモ](packages/client/libclient-py/demo/README-ja.md)

README.md

@@ -16,6 +16,7 @@ This will enable businesses and engineers worldwide to use Python and JavaScript
 - `make`
 - `docker`
 - `docker-compose`
+- [`vscode-yaml-sort`](https://marketplace.visualstudio.com/items?itemName=PascalReitermann93.vscode-yaml-sort)
 
 ## Getting Started
 [The demonstration with libClient-py](packages/client/libclient-py/demo/README.md)

scripts/Makefile

@@ -23,6 +23,7 @@ downnet:
 test: upnet
 	# tで指定されたパス以下のtest.shを全て探してきて実行する
 	echo -n "" > result
+	find $(t) -type f -name "$(p)*test.sh"
 	find $(t) -type f -name "$(p)*test.sh" | xargs -t -I{} bash tester.sh {} $m &&\
 	test ! -s result && exit 1;\
 	EXIT_CODE=$$(awk '{s+=$$1} END {print s}' result) &&\
@@ -31,8 +32,9 @@ test: upnet
 debug: upnet
 	# tで指定されたパス以下のdebug.shを全て探してきて実行する
 	echo -n "" > result
+	find $(t) -type f -name "${p}*debug.sh"
 	if [ "$(t)" = "." ] && [ "$(p)" = "" ]; then p=full; fi; \
-	find $(t) -type f -name "$${p}*debug.sh" | xargs -t -I{} bash debugger.sh {} $m
+	find $(t) -type f -name "${p}*debug.sh" | xargs -t -I{} bash debugger.sh {} $m
 
 rm:
 	docker-compose rm -fs

scripts/beaver_triple_service/server/engine_to_bts/small_test.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 build() {
-    docker buildx bake -f docker-compose.yml small_bts --load
+    docker buildx bake $COMPOSE_FILES_OPT small_bts --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -count 1 -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/server/engine_to_bts/...)'
+    docker-compose $COMPOSE_FILES_OPT run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -count 1 -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/server/engine_to_bts/...)'
 }
 
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/beaver_triple_service/triple_generator/small_test.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 build() {
-    docker buildx bake -f docker-compose.yml small_bts --load
+    docker buildx bake $COMPOSE_FILES_OPT small_bts --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/triple_generator/...)'
+    docker-compose $COMPOSE_FILES_OPT run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/triple_generator/...)'
 }
 
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/beaver_triple_service/triple_store/small_test.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 build() {
-    docker buildx bake -f docker-compose.yml small_bts --load
+    docker buildx bake $COMPOSE_FILES_OPT small_bts --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/triple_store/...)'
+    docker-compose $COMPOSE_FILES_OPT run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/triple_store/...)'
 }
 
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/beaver_triple_service/utils/small_test.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 build() {
-    docker buildx bake -f docker-compose.yml small_bts --load
+    docker buildx bake $COMPOSE_FILES_OPT small_bts --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/utils/...)'
+    docker-compose $COMPOSE_FILES_OPT run small_bts /bin/sh -c 'cd /QuickMPC && go mod vendor && go test -cover -v $(go list github.com/acompany-develop/QuickMPC/packages/server/beaver_triple_service/utils/...)'
 }
 
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/computation_container/benchmark_test.sh

@@ -3,23 +3,23 @@
 
 # build処理を記述する関数
 build() {
-    docker buildx bake -f docker-compose.yml dev_bts benchmark_cc1 benchmark_cc2 benchmark_cc3 --load
+    docker buildx bake $COMPOSE_FILES_OPT dev_bts benchmark_cc1 benchmark_cc2 benchmark_cc3 --load
 }
 
 # runの前に実行されるsetup処理を記述する関数
 # INFO: runを冪等に実行できるように初期化を行う
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 # run処理を記述する関数
 # NOTE: この関数は例外的にワンライナーで書かなくて良い
 run() {
-    docker-compose -f docker-compose.yml up benchmark_cc1 benchmark_cc2 benchmark_cc3
+    docker-compose $COMPOSE_FILES_OPT up benchmark_cc1 benchmark_cc2 benchmark_cc3
 }
 
 # runの後に実行されるteardown処理を記述する関数
 # INFO: runの後に副作用を残さないように初期化を行う
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/computation_container/cc_debug.sh

@@ -1,12 +1,12 @@
 # packages/server/<path>/*_test.*の実行方法をTest/<path>/test.shに記述する
 build() {
-    docker buildx bake -f docker-compose.yml dev_bts dev_cc1 dev_cc2 dev_cc3 --load
+    docker buildx bake $COMPOSE_FILES_OPT dev_bts dev_cc1 dev_cc2 dev_cc3 --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose -f docker-compose.yml up dev_cc1 dev_cc2 dev_cc3
+    docker-compose $COMPOSE_FILES_OPT up dev_cc1 dev_cc2 dev_cc3
 }

scripts/computation_container/medium_test.sh

@@ -3,23 +3,23 @@
 
 # build処理を記述する関数
 build() {
-    docker buildx bake -f docker-compose.yml dev_bts medium_cc1 medium_cc2 medium_cc3 --load
+    docker buildx bake $COMPOSE_FILES_OPT dev_bts medium_cc1 medium_cc2 medium_cc3 --load
 }
 
 # runの前に実行されるsetup処理を記述する関数
 # INFO: runを冪等に実行できるように初期化を行う
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 # run処理を記述する関数
 # NOTE: この関数は例外的にワンライナーで書かなくて良い
 run() {
-    docker-compose -f docker-compose.yml up medium_cc1 medium_cc2 medium_cc3
+    docker-compose $COMPOSE_FILES_OPT up medium_cc1 medium_cc2 medium_cc3
 }
 
 # runの後に実行されるteardown処理を記述する関数
 # INFO: runの後に副作用を残さないように初期化を行う
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/computation_container/small_test.sh

@@ -3,23 +3,23 @@
 
 # build処理を記述する関数
 build() {
-    docker buildx bake -f docker-compose.yml small_cc --load
+    docker buildx bake $COMPOSE_FILES_OPT small_cc --load
 }
 
 # runの前に実行されるsetup処理を記述する関数
 # INFO: runを冪等に実行できるように初期化を行う
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 # run処理を記述する関数
 # NOTE: この関数は例外的にワンライナーで書かなくて良い
 run() {
-    docker-compose -f docker-compose.yml run -T small_cc /bin/sh -c "cd /QuickMPC && bazel test //test/unit_test:all --config=debug --test_env=IS_TEST=true --test_output=all"
+    docker-compose $COMPOSE_FILES_OPT run -T small_cc /bin/sh -c "cd /QuickMPC && bazel test //test/unit_test:all --config=debug --test_env=IS_TEST=true --test_output=all"
 }
 
 # runの後に実行されるteardown処理を記述する関数
 # INFO: runの後に副作用を残さないように初期化を行う
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/debugger.sh

@@ -5,6 +5,15 @@
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
+# docker-composeの読み込み先を指定
+## ファイル名のパターンを指定
+pattern="docker-compose*.yml"
+## ファイル名の一覧を取得し、-fオプションを付けて連結する
+export COMPOSE_FILES_OPT=$(find . -name "$pattern" -exec printf -- '-f %s ' {} +)
+## 結果を表示する
+echo "COMPOSE_FILES_OPT = $COMPOSE_FILES_OPT"
+
+
 # 引数が正しくない時に発火する関数
 usage_exit() {
     echo "Usage: $1 'build' or 'run' or ''" 1>&2

scripts/demo/full_debug.sh

@@ -3,23 +3,23 @@
 
 # build処理を記述する関数
 build() {
-    docker buildx bake -f docker-compose.yml dev_bts dev_cc1 dev_cc2 dev_cc3 dev_mc1 dev_mc2 dev_mc3 --load
+    docker buildx bake $COMPOSE_FILES_OPT dev_bts dev_cc1 dev_cc2 dev_cc3 dev_mc1 dev_mc2 dev_mc3 --load
 }
 
 # runの前に実行されるsetup処理を記述する関数
 # INFO: runを冪等に実行できるように初期化を行う
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 # run処理を記述する関数
 # NOTE: この関数は例外的にワンライナーで書かなくて良い
 run() {
-    docker-compose -f docker-compose.yml up dev_mc1 dev_mc2 dev_mc3
+    docker-compose $COMPOSE_FILES_OPT up dev_mc1 dev_mc2 dev_mc3
 }
 
 # runの後に実行されるteardown処理を記述する関数
 # INFO: runの後に副作用を残さないように初期化を行う
 teardown() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }

scripts/demo/send_share_debug.sh

@@ -1,12 +1,12 @@
 
 build() {
-    docker buildx bake -f docker-compose.yml send_share_mc1 send_share_mc2 send_share_mc3 --load
+    docker buildx bake $COMPOSE_FILES_OPT send_share_mc1 send_share_mc2 send_share_mc3 --load
 }
 
 setup() {
-    docker-compose -f docker-compose.yml down -v
+    docker-compose $COMPOSE_FILES_OPT down -v
 }
 
 run() {
-    docker-compose -f docker-compose.yml up send_share_mc1 send_share_mc2 send_share_mc3
+    docker-compose $COMPOSE_FILES_OPT up send_share_mc1 send_share_mc2 send_share_mc3
 }

scripts/docker-compose.core.yml

@@ -0,0 +1,11 @@
+version: '3.3'
+volumes:
+  resultdb1: {}
+  resultdb2: {}
+  resultdb3: {}
+  sharedb1: {}
+  sharedb2: {}
+  sharedb3: {}
+networks:
+  shared-network:
+    external: true