Skip to content

Commit 2627d33

Browse files
zehao-intelzehao-intel
authored
Update INC 3.0 New API for Keras Static Quant (#1400)
Signed-off-by: zehao-intel <zehao.huang@intel.com> Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent 8160c71 commit 2627d33

31 files changed

+2518
-16
lines changed

‎.azure-pipelines/scripts/install_nc.sh

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
#!/bin/bash
22

3+
echo -e "\n Install Neural Compressor ... "
34
cd /neural-compressor
4-
python -m pip install --no-cache-dir -r requirements.txt
5-
python setup.py bdist_wheel
6-
pip install dist/neural_compressor*.whl
5+
if [[ $1 = *"3x_pt" ]]; then
6+
python -m pip install --no-cache-dir -r requirements_pt.txt
7+
python setup.py pt bdist_wheel
8+
pip install dist/neural_compressor*.whl
9+
elif [[ $1 = *"3x_tf" ]]; then
10+
python -m pip install --no-cache-dir -r requirements_tf.txt
11+
python setup.py tf bdist_wheel
12+
pip install dist/neural_compressor*.whl
13+
else
14+
python -m pip install --no-cache-dir -r requirements.txt
15+
python setup.py 2x bdist_wheel
16+
pip install dist/neural_compressor*.whl
17+
fi
18+
19+
echo -e "\n pip list after install Neural Compressor ... "
720
pip list

‎.azure-pipelines/scripts/ut/collect_log_3x.sh renamed to ‎.azure-pipelines/scripts/ut/3x/collect_log_3x.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
source /neural-compressor/.azure-pipelines/scripts/change_color.sh
22

33
pip install coverage
4-
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.${1}
4+
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.${1}
55
coverage_log="/neural-compressor/log_dir/coverage_log"
66
coverage_log_base="/neural-compressor/log_dir/coverage_log_base"
77
coverage_compare="/neural-compressor/log_dir/coverage_compare.html"
@@ -23,7 +23,7 @@ git config --global --add safe.directory /neural-compressor
2323
git fetch
2424
git checkout master
2525
echo y | pip uninstall neural-compressor
26-
cd /neural-compressor/.azure-pipelines/scripts && bash install_nc.sh
26+
cd /neural-compressor/.azure-pipelines/scripts && bash install_nc.sh ${1}
2727

2828
coverage erase
2929
cd /neural-compressor/log_dir

‎.azure-pipelines/scripts/ut/coverage.pt renamed to ‎.azure-pipelines/scripts/ut/3x/coverage.3x_pt

File renamed without changes.

‎.azure-pipelines/scripts/ut/3x/coverage.3x_tf

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[run]
2+
branch = True
3+
4+
[report]
5+
include =
6+
*/neural_compressor/common/*
7+
*/neural_compressor/tensorflow/*
8+
exclude_lines =
9+
pragma: no cover
10+
raise NotImplementedError
11+
raise TypeError
12+
if self.device == "gpu":
13+
if device == "gpu":
14+
except ImportError:
15+
except Exception as e:

‎.azure-pipelines/scripts/ut/run_3x_pt.sh renamed to ‎.azure-pipelines/scripts/ut/3x/run_3x_pt.sh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
11
#!/bin/bash
2-
python -c "import neural_compressor as nc;print(nc.version.__version__)"
2+
python -c "import neural_compressor as nc"
33
test_case="run 3x Torch"
44
echo "${test_case}"
55

66
# install requirements
77
echo "set up UT env..."
8-
pip install -r /neural-compressor/requirements_pt.txt
98
pip install transformers
109
pip install coverage
1110
pip install pytest
1211
pip list
1312

14-
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.pt
15-
lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))')
13+
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_pt
14+
inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
1615
cd /neural-compressor/test || exit 1
17-
find ./3x/torch/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
16+
find ./3x/torch/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
1817

1918
LOG_DIR=/neural-compressor/log_dir
2019
mkdir -p ${LOG_DIR}

‎.azure-pipelines/scripts/ut/3x/run_3x_tf.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
python -c "import neural_compressor as nc"
3+
test_case="run 3x TensorFlow"
4+
echo "${test_case}"
5+
6+
# install requirements
7+
echo "set up UT env..."
8+
pip install coverage
9+
pip install pytest
10+
pip list
11+
12+
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_tf
13+
inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
14+
cd /neural-compressor/test || exit 1
15+
find ./3x/tensorflow/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
16+
17+
LOG_DIR=/neural-compressor/log_dir
18+
mkdir -p ${LOG_DIR}
19+
ut_log_name=${LOG_DIR}/ut_3x_tf.log
20+
21+
echo "cat run.sh..."
22+
sort run.sh -o run.sh
23+
cat run.sh | tee ${ut_log_name}
24+
echo "------UT start-------"
25+
bash -x run.sh 2>&1 | tee -a ${ut_log_name}
26+
cp .coverage ${LOG_DIR}/.coverage
27+
28+
echo "------UT end -------"
29+
30+
if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] || [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
31+
echo "Find errors in UT test, please check the output..."
32+
exit 1
33+
fi
34+
echo "UT finished successfully! "

‎.azure-pipelines/scripts/ut/coverage.file

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ omit =
1717
*/neural_compressor/template/*
1818
*/neural_compressor/common/*
1919
*/neural_compressor/torch/*
20+
*/neural_compressor/tensorflow/*
2021
exclude_lines =
2122
pragma: no cover
2223
raise NotImplementedError

‎.azure-pipelines/template/ut-template.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ steps:
3030

3131
- script: |
3232
docker exec ${{ parameters.utContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \
33-
&& bash install_nc.sh \
33+
&& bash install_nc.sh ${{ parameters.utScriptFileName }} \
3434
&& bash ut/${{ parameters.utScriptFileName }}.sh ${{ parameters.utTestMode }}"
3535
displayName: "Run UT"
3636

‎.azure-pipelines/ut-3x-pt.yml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ pr:
1212
- neural_compressor/torch
1313
- test/3x/torch
1414
- setup.py
15-
- requirements.txt
1615
- requirements_pt.txt
1716

1817
pool: ICX-16C
@@ -36,7 +35,7 @@ stages:
3635
- template: template/ut-template.yml
3736
parameters:
3837
dockerConfigName: "commonDockerConfig"
39-
utScriptFileName: "run_3x_pt"
38+
utScriptFileName: "3x/run_3x_pt"
4039
uploadPath: $(UPLOAD_PATH)
4140
utArtifact: "ut_coverage_3x"
4241

@@ -51,7 +50,7 @@ stages:
5150
- template: template/ut-template.yml
5251
parameters:
5352
dockerConfigName: "gitCloneDockerConfig"
54-
utScriptFileName: "run_3x_pt"
53+
utScriptFileName: "3x/run_3x_pt"
5554
uploadPath: $(UPLOAD_PATH)
5655
utArtifact: "ut_coverage_3x_baseline"
5756
repo: $(REPO)
@@ -87,8 +86,8 @@ stages:
8786
docker ps
8887
echo "--- collect logs ---"
8988
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
90-
&& bash install_nc.sh \
91-
&& bash ut/collect_log_3x.sh pt"
89+
&& bash install_nc.sh 3x_pt \
90+
&& bash ut/3x/collect_log_3x.sh 3x_pt"
9291
displayName: "collect logs"
9392
9493
- task: PublishPipelineArtifact@1

‎.azure-pipelines/ut-3x-tf.yml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
trigger: none
2+
3+
pr:
4+
autoCancel: true
5+
drafts: false
6+
branches:
7+
include:
8+
- master
9+
paths:
10+
include:
11+
- neural_compressor/common
12+
- neural_compressor/tensorflow
13+
- test/3x/tensorflow
14+
- setup.py
15+
- requirements_tf.txt
16+
17+
pool: ICX-16C
18+
19+
variables:
20+
IMAGE_NAME: "neural-compressor"
21+
IMAGE_TAG: "py38"
22+
UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
23+
DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
24+
ARTIFACT_NAME: "UT_coverage_report_3x_tf"
25+
REPO: $(Build.Repository.Uri)
26+
27+
stages:
28+
- stage: TensorFlow
29+
displayName: Unit Test 3x TensorFlow
30+
dependsOn: []
31+
jobs:
32+
- job:
33+
displayName: Unit Test 3x TensorFlow
34+
steps:
35+
- template: template/ut-template.yml
36+
parameters:
37+
dockerConfigName: "commonDockerConfig"
38+
utScriptFileName: "3x/run_3x_tf"
39+
uploadPath: $(UPLOAD_PATH)
40+
utArtifact: "ut_coverage_3x"
41+
42+
43+
- stage: TensorFlow_baseline
44+
displayName: Unit Test 3x TensorFlow baseline
45+
dependsOn: []
46+
jobs:
47+
- job:
48+
displayName: Unit Test 3x TensorFlow baseline
49+
steps:
50+
- template: template/ut-template.yml
51+
parameters:
52+
dockerConfigName: "gitCloneDockerConfig"
53+
utScriptFileName: "3x/run_3x_tf"
54+
uploadPath: $(UPLOAD_PATH)
55+
utArtifact: "ut_coverage_3x_baseline"
56+
repo: $(REPO)
57+
58+
- stage: Coverage
59+
displayName: "Coverage Combine"
60+
pool:
61+
vmImage: "ubuntu-latest"
62+
dependsOn: [TensorFlow, TensorFlow_baseline]
63+
jobs:
64+
- job: CollectDatafiles
65+
steps:
66+
- script: |
67+
if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
68+
docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
69+
fi
70+
docker images | grep -i ${IMAGE_NAME}
71+
if [[ $? -ne 0 ]]; then
72+
echo "NO Such Repo"
73+
exit 1
74+
fi
75+
displayName: "Build develop docker image"
76+
77+
- task: DownloadPipelineArtifact@2
78+
inputs:
79+
artifact:
80+
path: $(DOWNLOAD_PATH)
81+
82+
- script: |
83+
echo "--- create container ---"
84+
docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
85+
echo "--- docker ps ---"
86+
docker ps
87+
echo "--- collect logs ---"
88+
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
89+
&& bash install_nc.sh 3x_tf \
90+
&& bash ut/3x/collect_log_3x.sh 3x_tf"
91+
displayName: "collect logs"
92+
93+
- task: PublishPipelineArtifact@1
94+
condition: succeededOrFailed()
95+
inputs:
96+
targetPath: $(UPLOAD_PATH)
97+
artifact: $(ARTIFACT_NAME)
98+
publishLocation: "pipeline"
99+
100+
- task: Bash@3
101+
condition: always()
102+
inputs:
103+
targetType: "inline"
104+
script: |
105+
docker exec collectLogs bash -c "rm -fr /neural-compressor/* && rm -fr /neural-compressor/.* || true"
106+
displayName: "Docker clean up"

0 commit comments

Comments
 (0)