refactor example makefile for the seperated prepare phases

docs/docs/tutorials/ucf101.md

@@ -23,7 +23,7 @@ From this example, we can learn the following skills:
 
 ```shell
 cd starwhale/example/ucf101
-make prepare-data
+make prepare
 ```
 
 ## Build Runtime With Starwhale

docs/i18n/zh/docusaurus-plugin-content-docs/current/tutorials/ucf101.md

@@ -53,7 +53,7 @@ make train
 由于此数据集较大训练时间较长，也可以跳过训练步骤，使用已经准备好的模型进行接下来的评测。
 
 ```shell
-make prepare-data
+make prepare
 ```
 
 ## 2.Starwhale的模型评测过程

example/PennFudanPed/Makefile

@@ -5,17 +5,15 @@ train:
 
 .POHNY: download-data
 download-data:
-	rm -rf data
-	mkdir -p data
-	curl -o data/pfp.zip https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip
-	unzip data/pfp.zip -d data
-	rm -rf data/pfp.zip
-
-.POHNY: prepare-data
-prepare-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/PennFudanPed.zip ] || ([ -z $(CN) ] && curl -o data/PennFudanPed.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/PennFudanPed/PennFudanPed.zip ) || curl -o data/PennFudanPed.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/PennFudanPed/PennFudanPed.zip
 	[ -d data/PennFudanPed ] || unzip data/PennFudanPed.zip -d data
+
+.POHNY: download-model
+download-model:
 	[ -d models ] ||  mkdir -p models
 	[ -f models/mcrnn.pth ] || ([ -z $(CN) ] && curl -o models/mcrnn.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mcrnn_fudan.pth ) || curl -o models/mcrnn.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mcrnn_fudan.pth
+
+.POHNY: prepare
+prepare: download-model download-data
 	python3 -m pip install -r requirements.txt

example/cifar10/Makefile

@@ -3,12 +3,17 @@ train:
 	mkdir -p models data
 	python3 cifar/train.py
 
+.POHNY: download-model
+download-model:
+	[ -d models ] ||  mkdir -p models
+	[ -f models/cifar_net.pth ] || ([ -z $(CN) ] && curl -o models/cifar_net.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/cifar_net.pth) || curl -o models/cifar_net.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/cifar_net.pth
 
-.POHNY: prepare-data
-prepare-data:
+.POHNY: download-data
+download-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/cifar-10-python.tar.gz ] || ([ -z $(CN) ] && curl -o data/cifar-10-python.tar.gz https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/cifar/cifar-10-python.tar.gz) || curl -o data/cifar-10-python.tar.gz https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/cifar/cifar-10-python.tar.gz
 	[ -d data/cifar-10-batches-py ] || tar -xzf data/cifar-10-python.tar.gz -C data
-	[ -d models ] ||  mkdir -p models
-	[ -f models/cifar_net.pth ] || ([ -z $(CN) ] && curl -o models/cifar_net.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/cifar_net.pth) || curl -o models/cifar_net.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/cifar_net.pth
+
+.POHNY: prepare
+prepare: download-model download-data
 	python3 -m pip install -r requirements.txt

example/mnist/Makefile

@@ -1,12 +1,18 @@
+.POHNY: train
 train:
 	python mnist/train.py
 
+.POHNY: download-model
+download-model:
+	[ -d models ] ||  mkdir -p models
+	[ -f models/mnist_cnn.pt ] || ([ -z $(CN) ] && curl -o models/mnist_cnn.pt https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mnist.pth) || curl -o models/mnist_cnn.pt https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mnist.pth
 
-.POHNY: prepare-data
-prepare-data:
+.POHNY: download-data
+download-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/t10k-images-idx3-ubyte ] || ([ -z $(CN) ] && curl -o data/t10k-images-idx3-ubyte https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/mnist/t10k-images-idx3-ubyte) || curl -o data/t10k-images-idx3-ubyte https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/mnist/t10k-images-idx3-ubyte
 	[ -f data/t10k-labels-idx1-ubyte ] || ([ -z $(CN) ] && curl -o data/t10k-labels-idx1-ubyte https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/mnist/t10k-labels-idx1-ubyte) || curl -o data/t10k-labels-idx1-ubyte https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/mnist/t10k-labels-idx1-ubyte
-	[ -d models ] ||  mkdir -p models
-	[ -f models/mnist_cnn.pt ] || ([ -z $(CN) ] && curl -o models/mnist_cnn.pt https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mnist.pth) || curl -o models/mnist_cnn.pt https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mnist.pth
+
+.POHNY: prepare
+prepare: download-model download-model
 	python3 -m pip install -r requirements.txt

example/nmt/Makefile

@@ -3,24 +3,20 @@ train:
 	mkdir -p models
 	python3 main.py
 
+.POHNY: download-model
+download-model:
+	[ -d models ] ||  mkdir -p models
+	[ -f models/decoder.pth ] || ([ -z $(CN) ] && curl -o models/decoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/decoder.pth) || curl -o models/decoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/decoder.pth
+	[ -f models/encoder.pth ] || ([ -z $(CN) ] && curl -o models/encoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/encoder.pth) || curl -o models/encoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/encoder.pth
+	[ -f models/vocab_eng-fra.bin ] || ([ -z $(CN) ] && curl -o models/vocab_eng-fra.bin https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/vocab_eng-fra.bin) || curl -o models/vocab_eng-fra.bin https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/vocab_eng-fra.bin
+
 .POHNY: download-data
 download-data:
-	rm -rf data
-	mkdir -p data
-	curl -o data/fra-eng.zip https://www.manythings.org/anki/fra-eng.zip
-	unzip data/fra-eng.zip -d data
-	rm -rf data/fra-eng.zip
-	shuf -n 1000 data/fra.txt > data/fra-test.txt
-
-.POHNY: prepare-data
-prepare-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/fra-eng.zip ] || ([ -z $(CN) ] && curl -o data/fra-eng.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/nmt/fra-eng.zip) || curl -o data/fra-eng.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/nmt/fra-eng.zip
 	[ -f data/fra.txt ] || unzip data/fra-eng.zip -d data
 	[ -f data/fra-test.txt ] || shuf -n 1000 data/fra.txt > data/fra-test.txt
-	[ -d models ] ||  mkdir -p models
-	[ -f models/decoder.pth ] || ([ -z $(CN) ] && curl -o models/decoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/decoder.pth) || curl -o models/decoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/decoder.pth
-	[ -f models/encoder.pth ] || ([ -z $(CN) ] && curl -o models/encoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/encoder.pth) || curl -o models/encoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/encoder.pth
-	[ -f models/vocab_eng-fra.bin ] || ([ -z $(CN) ] && curl -o models/vocab_eng-fra.bin https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/vocab_eng-fra.bin) || curl -o models/vocab_eng-fra.bin https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/vocab_eng-fra.bin
-	python3 -m pip install -r requirements.txt
 
+.POHNY: prepare
+prepare: download-model download-data
+	python3 -m pip install -r requirements.txt

example/speech_command/Makefile

@@ -3,13 +3,19 @@ train:
 	mkdir -p models data
 	python3 sc/train.py
 
-.POHNY: prepare-data
-prepare-data:
+.POHNY: download-model
+download-model:
+	[ -d models ] ||  mkdir -p models
+	[ -f models/m5.pth ] || ([ -z $(CN) ] && curl -o models/m5.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/speech_cmd_m5.pth) || curl -o models/m5.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/speech_cmd_m5.pth
+
+.POHNY: download-data
+download-data:
 	[ -d data/SpeechCommands/speech_commands_v0.02 ] ||  mkdir -p data/SpeechCommands/speech_commands_v0.02
 	[ -f data/speech_commands_v0.02.tar.gz ] || ([ -z $(CN) ] && curl -o data/speech_commands_v0.02.tar.gz https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/SpeechCommands/speech_commands_v0.02.tar.gz) || curl -o data/speech_commands_v0.02.tar.gz https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/SpeechCommands/speech_commands_v0.02.tar.gz
 	[ -f data/SpeechCommands/speech_commands_v0.02/validation_list.txt ] || tar -xzf data/speech_commands_v0.02.tar.gz -C data/SpeechCommands/speech_commands_v0.02
-	[ -d models ] ||  mkdir -p models
-	[ -f models/m5.pth ] || ([ -z $(CN) ] && curl -o models/m5.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/speech_cmd_m5.pth) || curl -o models/m5.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/speech_cmd_m5.pth
+
+.POHNY: prepare
+prepare: download-model download-data
 	python3 -m pip install -r requirements.txt
 
 

example/text_cls_AG_NEWS/Makefile

@@ -5,17 +5,16 @@ train:
 
 .POHNY: download-data
 download-data:
-	rm -rf data
-	mkdir -p data
-	curl -o data/train.csv https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv/train.csv
-	curl -o data/test.csv https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv/test.csv
-
-.POHNY: prepare-data
-prepare-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/test.csv ] || ([ -z $(CN) ] && curl -o data/test.csv https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/ag_news/data/test.csv) || curl -o data/test.csv https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/ag_news/data/test.csv
+
+.POHNY: download-model
+download-model:
 	[ -d models ] ||  mkdir -p models
 	[ -f models/model.i ] || ([ -z $(CN) ] && curl -o models/model.i https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ag_news/models/model.i) || curl -o models/model.i https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ag_news/models/model.i
 	[ -f models/vocab.i ] || ([ -z $(CN) ] && curl -o models/vocab.i https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ag_news/models/vocab.i) || curl -o models/vocab.i https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ag_news/models/vocab.i
+
+.POHNY: prepare
+prepare: download-model download-data
 	python3 -m pip install -r requirements.txt
 

example/ucf101/Makefile

@@ -3,8 +3,8 @@ train:
 	mkdir -p models
 	python3 ucf101/train.py
 
-.POHNY: download-data
-download-data:
+.POHNY: build-raw-data
+build-raw-data:
 	rm -rf data
 	mkdir -p data
 	wget http://www.crcv.ucf.edu/data/UCF101/UCF101.rar --no-check-certificate -P data
@@ -16,14 +16,20 @@ download-data:
 	shuf data/all_list.txt -n 1000 -o data/validation_list.txt
 	shuf data/all_list.txt -n 200 -o data/test_list.txt
 
-.POHNY: prepare-data
-prepare-data:
+.POHNY: download-data
+download-data:
 	[ -d data ] ||  mkdir -p data
 	[ -f data/data.zip ] || ([ -z $(CN) ] && curl -o data/data.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/ucf101/data.zip) || curl -o data/data.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/ucf101/data.zip
 	[ -f data/test_list.txt ] || unzip data/data.zip -d .
-	[ -d data/UCF-101 ] || mv data/UCF-101-WEBM data/UCF-101
+
+.POHNY: download-model
+download-model:
 	[ -d models ] ||  mkdir -p models
 	[ -f models/PyTorch-MFNet_ep-0000.pth ] || ([ -z $(CN) ] && curl -o models/PyTorch-MFNet_ep-0000.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ucf101.pth) || curl -o models/PyTorch-MFNet_ep-0000.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ucf101.pth
+
+.POHNY: prepare
+prepare: download-model download-data
+[ -d data/UCF-101 ] || mv data/UCF-101-WEBM data/UCF-101
 	python3 -m pip install -r requirements.txt
 
 .POHNY: show-labels

scripts/client_test/cli_test.py

@@ -276,7 +276,7 @@ def test_all(self) -> None:
         for name, expl in EXAMPLES.items():
             logger.info(f"preparing data for {expl}")
             rc = subprocess.call(
-                ["make", "CN=1", "prepare-data"],
+                ["make", "CN=1", "prepare"],
                 cwd=expl["workdir"],
             )
             if rc != 0:
@@ -321,7 +321,7 @@ def test_expl(self, expl_name: str) -> None:
         workdir_ = str(expl["workdir"])
 
         p = subprocess.Popen(
-            ["make", "prepare-data"],
+            ["make", "prepare"],
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             cwd=workdir_,