diff --git a/docs/docs/tutorials/ucf101.md b/docs/docs/tutorials/ucf101.md index 9b4e834916..cad012324a 100644 --- a/docs/docs/tutorials/ucf101.md +++ b/docs/docs/tutorials/ucf101.md @@ -23,7 +23,7 @@ From this example, we can learn the following skills: ```shell cd starwhale/example/ucf101 -make prepare-data +make prepare ``` ## Build Runtime With Starwhale diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/tutorials/ucf101.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/tutorials/ucf101.md index e190e4b27e..d7dde9786d 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/tutorials/ucf101.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/tutorials/ucf101.md @@ -53,7 +53,7 @@ make train 由于此数据集较大训练时间较长，也可以跳过训练步骤，使用已经准备好的模型进行接下来的评测。 ```shell -make prepare-data +make prepare ``` ## 2.Starwhale的模型评测过程 diff --git a/example/PennFudanPed/Makefile b/example/PennFudanPed/Makefile index 198ffc4b22..624a47799c 100644 --- a/example/PennFudanPed/Makefile +++ b/example/PennFudanPed/Makefile @@ -5,17 +5,15 @@ train: .POHNY: download-data download-data: - rm -rf data - mkdir -p data - curl -o data/pfp.zip https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip - unzip data/pfp.zip -d data - rm -rf data/pfp.zip - -.POHNY: prepare-data -prepare-data: [ -d data ] || mkdir -p data [ -f data/PennFudanPed.zip ] || ([ -z $(CN) ] && curl -o data/PennFudanPed.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/PennFudanPed/PennFudanPed.zip ) || curl -o data/PennFudanPed.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/PennFudanPed/PennFudanPed.zip [ -d data/PennFudanPed ] || unzip data/PennFudanPed.zip -d data + +.POHNY: download-model +download-model: [ -d models ] || mkdir -p models [ -f models/mcrnn.pth ] || ([ -z $(CN) ] && curl -o models/mcrnn.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mcrnn_fudan.pth ) || curl -o models/mcrnn.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mcrnn_fudan.pth + +.POHNY: prepare +prepare: download-model download-data python3 -m pip install -r requirements.txt diff --git a/example/cifar10/Makefile b/example/cifar10/Makefile index a41ee61076..a7a428de77 100644 --- a/example/cifar10/Makefile +++ b/example/cifar10/Makefile @@ -3,12 +3,17 @@ train: mkdir -p models data python3 cifar/train.py +.POHNY: download-model +download-model: + [ -d models ] || mkdir -p models + [ -f models/cifar_net.pth ] || ([ -z $(CN) ] && curl -o models/cifar_net.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/cifar_net.pth) || curl -o models/cifar_net.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/cifar_net.pth -.POHNY: prepare-data -prepare-data: +.POHNY: download-data +download-data: [ -d data ] || mkdir -p data [ -f data/cifar-10-python.tar.gz ] || ([ -z $(CN) ] && curl -o data/cifar-10-python.tar.gz https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/cifar/cifar-10-python.tar.gz) || curl -o data/cifar-10-python.tar.gz https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/cifar/cifar-10-python.tar.gz [ -d data/cifar-10-batches-py ] || tar -xzf data/cifar-10-python.tar.gz -C data - [ -d models ] || mkdir -p models - [ -f models/cifar_net.pth ] || ([ -z $(CN) ] && curl -o models/cifar_net.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/cifar_net.pth) || curl -o models/cifar_net.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/cifar_net.pth + +.POHNY: prepare +prepare: download-model download-data python3 -m pip install -r requirements.txt diff --git a/example/mnist/Makefile b/example/mnist/Makefile index 58aa4940a0..c08e377e76 100644 --- a/example/mnist/Makefile +++ b/example/mnist/Makefile @@ -1,12 +1,18 @@ +.POHNY: train train: python mnist/train.py +.POHNY: download-model +download-model: + [ -d models ] || mkdir -p models + [ -f models/mnist_cnn.pt ] || ([ -z $(CN) ] && curl -o models/mnist_cnn.pt https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mnist.pth) || curl -o models/mnist_cnn.pt https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mnist.pth -.POHNY: prepare-data -prepare-data: +.POHNY: download-data +download-data: [ -d data ] || mkdir -p data [ -f data/t10k-images-idx3-ubyte ] || ([ -z $(CN) ] && curl -o data/t10k-images-idx3-ubyte https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/mnist/t10k-images-idx3-ubyte) || curl -o data/t10k-images-idx3-ubyte https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/mnist/t10k-images-idx3-ubyte [ -f data/t10k-labels-idx1-ubyte ] || ([ -z $(CN) ] && curl -o data/t10k-labels-idx1-ubyte https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/mnist/t10k-labels-idx1-ubyte) || curl -o data/t10k-labels-idx1-ubyte https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/mnist/t10k-labels-idx1-ubyte - [ -d models ] || mkdir -p models - [ -f models/mnist_cnn.pt ] || ([ -z $(CN) ] && curl -o models/mnist_cnn.pt https://starwhale-examples.s3.us-west-1.amazonaws.com/model/mnist.pth) || curl -o models/mnist_cnn.pt https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/mnist.pth + +.POHNY: prepare +prepare: download-model download-model python3 -m pip install -r requirements.txt diff --git a/example/nmt/Makefile b/example/nmt/Makefile index 58b744c5bc..4125daf7ad 100644 --- a/example/nmt/Makefile +++ b/example/nmt/Makefile @@ -3,24 +3,20 @@ train: mkdir -p models python3 main.py +.POHNY: download-model +download-model: + [ -d models ] || mkdir -p models + [ -f models/decoder.pth ] || ([ -z $(CN) ] && curl -o models/decoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/decoder.pth) || curl -o models/decoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/decoder.pth + [ -f models/encoder.pth ] || ([ -z $(CN) ] && curl -o models/encoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/encoder.pth) || curl -o models/encoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/encoder.pth + [ -f models/vocab_eng-fra.bin ] || ([ -z $(CN) ] && curl -o models/vocab_eng-fra.bin https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/vocab_eng-fra.bin) || curl -o models/vocab_eng-fra.bin https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/vocab_eng-fra.bin + .POHNY: download-data download-data: - rm -rf data - mkdir -p data - curl -o data/fra-eng.zip https://www.manythings.org/anki/fra-eng.zip - unzip data/fra-eng.zip -d data - rm -rf data/fra-eng.zip - shuf -n 1000 data/fra.txt > data/fra-test.txt - -.POHNY: prepare-data -prepare-data: [ -d data ] || mkdir -p data [ -f data/fra-eng.zip ] || ([ -z $(CN) ] && curl -o data/fra-eng.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/nmt/fra-eng.zip) || curl -o data/fra-eng.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/nmt/fra-eng.zip [ -f data/fra.txt ] || unzip data/fra-eng.zip -d data [ -f data/fra-test.txt ] || shuf -n 1000 data/fra.txt > data/fra-test.txt - [ -d models ] || mkdir -p models - [ -f models/decoder.pth ] || ([ -z $(CN) ] && curl -o models/decoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/decoder.pth) || curl -o models/decoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/decoder.pth - [ -f models/encoder.pth ] || ([ -z $(CN) ] && curl -o models/encoder.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/encoder.pth) || curl -o models/encoder.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/encoder.pth - [ -f models/vocab_eng-fra.bin ] || ([ -z $(CN) ] && curl -o models/vocab_eng-fra.bin https://starwhale-examples.s3.us-west-1.amazonaws.com/model/nmt/vocab_eng-fra.bin) || curl -o models/vocab_eng-fra.bin https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/nmt/vocab_eng-fra.bin - python3 -m pip install -r requirements.txt +.POHNY: prepare +prepare: download-model download-data + python3 -m pip install -r requirements.txt diff --git a/example/speech_command/Makefile b/example/speech_command/Makefile index 57212663db..e528280a4b 100644 --- a/example/speech_command/Makefile +++ b/example/speech_command/Makefile @@ -3,13 +3,19 @@ train: mkdir -p models data python3 sc/train.py -.POHNY: prepare-data -prepare-data: +.POHNY: download-model +download-model: + [ -d models ] || mkdir -p models + [ -f models/m5.pth ] || ([ -z $(CN) ] && curl -o models/m5.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/speech_cmd_m5.pth) || curl -o models/m5.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/speech_cmd_m5.pth + +.POHNY: download-data +download-data: [ -d data/SpeechCommands/speech_commands_v0.02 ] || mkdir -p data/SpeechCommands/speech_commands_v0.02 [ -f data/speech_commands_v0.02.tar.gz ] || ([ -z $(CN) ] && curl -o data/speech_commands_v0.02.tar.gz https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/SpeechCommands/speech_commands_v0.02.tar.gz) || curl -o data/speech_commands_v0.02.tar.gz https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/SpeechCommands/speech_commands_v0.02.tar.gz [ -f data/SpeechCommands/speech_commands_v0.02/validation_list.txt ] || tar -xzf data/speech_commands_v0.02.tar.gz -C data/SpeechCommands/speech_commands_v0.02 - [ -d models ] || mkdir -p models - [ -f models/m5.pth ] || ([ -z $(CN) ] && curl -o models/m5.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/speech_cmd_m5.pth) || curl -o models/m5.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/speech_cmd_m5.pth + +.POHNY: prepare +prepare: download-model download-data python3 -m pip install -r requirements.txt diff --git a/example/text_cls_AG_NEWS/Makefile b/example/text_cls_AG_NEWS/Makefile index b36c3e2fb7..92e2b02ba6 100644 --- a/example/text_cls_AG_NEWS/Makefile +++ b/example/text_cls_AG_NEWS/Makefile @@ -5,17 +5,16 @@ train: .POHNY: download-data download-data: - rm -rf data - mkdir -p data - curl -o data/train.csv https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv/train.csv - curl -o data/test.csv https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv/test.csv - -.POHNY: prepare-data -prepare-data: [ -d data ] || mkdir -p data [ -f data/test.csv ] || ([ -z $(CN) ] && curl -o data/test.csv https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/ag_news/data/test.csv) || curl -o data/test.csv https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/ag_news/data/test.csv + +.POHNY: download-model +download-model: [ -d models ] || mkdir -p models [ -f models/model.i ] || ([ -z $(CN) ] && curl -o models/model.i https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ag_news/models/model.i) || curl -o models/model.i https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ag_news/models/model.i [ -f models/vocab.i ] || ([ -z $(CN) ] && curl -o models/vocab.i https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ag_news/models/vocab.i) || curl -o models/vocab.i https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ag_news/models/vocab.i + +.POHNY: prepare +prepare: download-model download-data python3 -m pip install -r requirements.txt diff --git a/example/ucf101/Makefile b/example/ucf101/Makefile index 5993bf00f5..bb8e5c8de8 100644 --- a/example/ucf101/Makefile +++ b/example/ucf101/Makefile @@ -3,8 +3,8 @@ train: mkdir -p models python3 ucf101/train.py -.POHNY: download-data -download-data: +.POHNY: build-raw-data +build-raw-data: rm -rf data mkdir -p data wget http://www.crcv.ucf.edu/data/UCF101/UCF101.rar --no-check-certificate -P data @@ -16,14 +16,20 @@ download-data: shuf data/all_list.txt -n 1000 -o data/validation_list.txt shuf data/all_list.txt -n 200 -o data/test_list.txt -.POHNY: prepare-data -prepare-data: +.POHNY: download-data +download-data: [ -d data ] || mkdir -p data [ -f data/data.zip ] || ([ -z $(CN) ] && curl -o data/data.zip https://starwhale-examples.s3.us-west-1.amazonaws.com/dataset/ucf101/data.zip) || curl -o data/data.zip https://starwhale-examples.oss-cn-beijing.aliyuncs.com/dataset/ucf101/data.zip [ -f data/test_list.txt ] || unzip data/data.zip -d . - [ -d data/UCF-101 ] || mv data/UCF-101-WEBM data/UCF-101 + +.POHNY: download-model +download-model: [ -d models ] || mkdir -p models [ -f models/PyTorch-MFNet_ep-0000.pth ] || ([ -z $(CN) ] && curl -o models/PyTorch-MFNet_ep-0000.pth https://starwhale-examples.s3.us-west-1.amazonaws.com/model/ucf101.pth) || curl -o models/PyTorch-MFNet_ep-0000.pth https://starwhale-examples.oss-cn-beijing.aliyuncs.com/model/ucf101.pth + +.POHNY: prepare +prepare: download-model download-data +[ -d data/UCF-101 ] || mv data/UCF-101-WEBM data/UCF-101 python3 -m pip install -r requirements.txt .POHNY: show-labels diff --git a/scripts/client_test/cli_test.py b/scripts/client_test/cli_test.py index 5624757731..cc6ed6199f 100644 --- a/scripts/client_test/cli_test.py +++ b/scripts/client_test/cli_test.py @@ -276,7 +276,7 @@ def test_all(self) -> None: for name, expl in EXAMPLES.items(): logger.info(f"preparing data for {expl}") rc = subprocess.call( - ["make", "CN=1", "prepare-data"], + ["make", "CN=1", "prepare"], cwd=expl["workdir"], ) if rc != 0: @@ -321,7 +321,7 @@ def test_expl(self, expl_name: str) -> None: workdir_ = str(expl["workdir"]) p = subprocess.Popen( - ["make", "prepare-data"], + ["make", "prepare"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=workdir_,