Skip to content

Commit dfd34c8

Browse files
authored
add tools for standardize formatting & sanity-check (#63)
* BLD: add auto format and sanity check tools * BLD: add ci requirements * CLN: fix bash code style * ENH: --incremental find files changed from master * BLD: add install_ci_dependency script * BLD: add makefile * CLN: minor fix * ENH: more robust way to check root dir * CLN: minor fix * CLN: replace unit-test by addons_cpu.sh * BUG: fix, local_config_tf is missing * DOC: hightlight make commands
1 parent bfaa4ed commit dfd34c8

File tree

13 files changed

+1442
-3
lines changed

13 files changed

+1442
-3
lines changed

CONTRIBUTING.md

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,31 @@ us**
5252
It is recommended that development is done in the latest
5353
`nightly-custom-op` docker image.
5454
```
55-
docker run --rm -it -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:nightly-custom-op /bin/bash
55+
docker run --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:nightly-custom-op /bin/bash
5656
```
5757

58+
Try those commands below:
59+
60+
0. Format codes automatically: `make code-format`
61+
1. Sanity check: `make sanity-check`
62+
2. Run unit test: `make unit-test`
63+
3. All of the above: `make`
64+
65+
5866
## Code Testing
5967
#### CI Testing
6068
We're in the process of setting up our nightly CI testing. Because this
6169
project will contain CUDA kernels, we need to make sure that the
6270
hardware will be available from our CI provider.
6371

6472
#### Locally Testing
73+
74+
```
75+
docker run --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:nightly-custom-op make unit-test
76+
```
77+
78+
or run manually:
79+
6580
```
6681
./configure.sh # Links project with TensorFlow dependency
6782

ci_testing/addons_cpu.sh

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@
1414
# limitations under the License.
1515
#
1616
# ==============================================================================
17+
# Make sure we're in the project root path.
18+
SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
19+
ROOT_DIR=$( cd "$SCRIPT_DIR/.." && pwd -P )
20+
if [[ ! -d "tensorflow_addons" ]]; then
21+
echo "ERROR: PWD: $PWD is not project root"
22+
exit 1
23+
fi
1724

1825
set -x
1926

@@ -27,12 +34,13 @@ export CC_OPT_FLAGS='-mavx'
2734
export TF_NEED_CUDA=0 # TODO: Verify this is used in GPU custom-op
2835

2936
export PYTHON_BIN_PATH=`which python`
30-
/bin/bash configure.sh
37+
# Use default configuration here.
38+
yes 'y' | ./configure.sh
3139

3240
## Run bazel test command. Double test timeouts to avoid flakes.
3341
bazel test -c opt -k \
3442
--jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
3543
--test_output=errors --local_test_jobs=8 \
3644
//tensorflow_addons/...
3745

38-
exit $?
46+
exit $?

makefile

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# ==============================================================================
15+
.PHONY: all
16+
17+
all: code-format sanity-check unit-test
18+
19+
# TODO: install those dependencies in docker image (dockerfile).
20+
install-ci-dependency:
21+
bash tools/ci_build/install/install_ci_dependency.sh --quiet
22+
23+
code-format: install-ci-dependency
24+
bash tools/ci_build/code_format.sh --incremental --in-place
25+
26+
sanity-check: install-ci-dependency
27+
bash tools/ci_build/ci_sanity.sh --incremental
28+
29+
unit-test:
30+
bash ci_testing/addons_cpu.sh
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
#
17+
# Common Bash functions used by build scripts
18+
19+
COLOR_NC='\033[0m'
20+
COLOR_BOLD='\033[1m'
21+
COLOR_LIGHT_GRAY='\033[0;37m'
22+
COLOR_GREEN='\033[0;32m'
23+
COLOR_RED='\033[0;31m'
24+
25+
die() {
26+
# Print a message and exit with code 1.
27+
#
28+
# Usage: die <error_message>
29+
# e.g., die "Something bad happened."
30+
31+
echo $@
32+
exit 1
33+
}
34+
35+
num_cpus() {
36+
# Get the number of CPUs
37+
N_CPUS=$(grep -c ^processor /proc/cpuinfo)
38+
if [[ -z ${N_CPUS} ]]; then
39+
die "ERROR: Unable to determine the number of CPUs"
40+
fi
41+
42+
echo ${N_CPUS}
43+
}
44+
45+
# List files changed (i.e., added, or revised).
46+
# Usage: get_changed_files_from_master_branch
47+
get_changed_files_from_master_branch() {
48+
git diff origin/master --diff-filter=d --name-only "$@"
49+
}
50+
51+
# List bazel files changed that still exist,
52+
# i.e., not removed.
53+
# Usage: get_bazel_files_to_check [--incremental]
54+
get_bazel_files_to_check() {
55+
if [[ "$1" == "--incremental" ]]; then
56+
get_changed_files_from_master_branch -- 'BUILD*'
57+
elif [[ -z "$1" ]]; then
58+
find . -name 'BUILD*'
59+
else
60+
die "Found unsupported args: $@ for get_bazel_files_to_check."
61+
fi
62+
}
63+
64+
# List python files changed that still exist,
65+
# i.e., not removed.
66+
# Usage: get_py_files_to_check [--incremental]
67+
get_py_files_to_check() {
68+
if [[ "$1" == "--incremental" ]]; then
69+
get_changed_files_from_master_branch -- '*.py'
70+
elif [[ -z "$1" ]]; then
71+
find . -name '*.py'
72+
else
73+
die "Found unsupported args: $@ for get_py_files_to_check."
74+
fi
75+
}
76+
77+
# List .h|.cc files changed that still exist,
78+
# i.e., not removed.
79+
# Usage: get_clang_files_to_check [--incremental]
80+
get_clang_files_to_check() {
81+
if [[ "$1" == "--incremental" ]]; then
82+
get_changed_files_from_master_branch -- '*.h' '*.cc'
83+
elif [[ -z "$1" ]]; then
84+
find . -name '*.h' -o -name '*.cc'
85+
else
86+
die "Found unsupported args: $@ for get_clang_files_to_check."
87+
fi
88+
}
89+
90+
realpath() {
91+
# Get the real path of a file
92+
# Usage: realpath <file_path>
93+
94+
if [[ $# != "1" ]]; then
95+
die "realpath: incorrect usage"
96+
fi
97+
98+
[[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
99+
}
100+
101+
to_lower () {
102+
# Convert string to lower case.
103+
# Usage: to_lower <string>
104+
105+
echo "$1" | tr '[:upper:]' '[:lower:]'
106+
}
107+
108+
calc_elapsed_time() {
109+
# Calculate elapsed time. Takes nanosecond format input of the kind output
110+
# by date +'%s%N'
111+
#
112+
# Usage: calc_elapsed_time <START_TIME> <END_TIME>
113+
114+
if [[ $# != "2" ]]; then
115+
die "calc_elapsed_time: incorrect usage"
116+
fi
117+
118+
START_TIME=$1
119+
END_TIME=$2
120+
121+
if [[ ${START_TIME} == *"N" ]]; then
122+
# Nanosecond precision not available
123+
START_TIME=$(echo ${START_TIME} | sed -e 's/N//g')
124+
END_TIME=$(echo ${END_TIME} | sed -e 's/N//g')
125+
ELAPSED="$(expr ${END_TIME} - ${START_TIME}) s"
126+
else
127+
ELAPSED="$(expr $(expr ${END_TIME} - ${START_TIME}) / 1000000) ms"
128+
fi
129+
130+
echo ${ELAPSED}
131+
}

0 commit comments

Comments
 (0)