Skip to content

Commit a6b20d3

Browse files
yf225yf225
authored
Fix tutorial generation issues (pytorch#373)
* Revert "Make all tutorials run their normal length" This reverts commit 4543f6a. * Fix tutorial generation issues * fix indent * make cifar10_tutorial.py even faster * rename tensor_tutorial.py to tensor_tutorial_old.py * fix bugs * Revert "make cifar10_tutorial.py even faster" This reverts commit 33d3a90. * fix bug * Revert "Revert "Make all tutorials run their normal length"" This reverts commit 9664852.
1 parent fe4cbcd commit a6b20d3

10 files changed

+131
-28
lines changed

‎.jenkins/build.sh

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,17 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
6565
rm beginner_source/aws_distributed_training_tutorial.py || true
6666

6767
# Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials
68+
# IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename.
6869
export WORKER_ID=$(echo "${JOB_BASE_NAME}" | tr -dc '0-9')
6970
count=0
71+
FILES_TO_RUN=()
7072
for filename in $(find beginner_source/ -name '*.py' -not -path '*/data/*'); do
7173
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
7274
echo "Removing runnable code from "$filename
7375
python $DIR/remove_runnable_code.py $filename $filename
7476
else
7577
echo "Keeping "$filename
78+
FILES_TO_RUN+=($(basename $filename .py))
7679
fi
7780
count=$((count+1))
7881
done
@@ -82,6 +85,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
8285
python $DIR/remove_runnable_code.py $filename $filename
8386
else
8487
echo "Keeping "$filename
88+
FILES_TO_RUN+=($(basename $filename .py))
8589
fi
8690
count=$((count+1))
8791
done
@@ -91,28 +95,58 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
9195
python $DIR/remove_runnable_code.py $filename $filename
9296
else
9397
echo "Keeping "$filename
98+
FILES_TO_RUN+=($(basename $filename .py))
9499
fi
95100
count=$((count+1))
96101
done
102+
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
97103

98104
# Step 3: Run `make docs` to generate HTML files and static files for these tutorials
99105
make docs
100106

101-
# Step 4: Remove all HTML files generated from Python files that don't contain runnable code
102-
for filename in $(find docs/beginner -name '*.html'); do
103-
python $DIR/delete_html_file_with_runnable_code_removed.py $filename
107+
# Step 4: If any of the generated files are not related the tutorial files we want to run,
108+
# then we remove them
109+
for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.html'); do
110+
file_basename=$(basename $filename .html)
111+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
112+
rm $filename
113+
fi
104114
done
105-
for filename in $(find docs/intermediate -name '*.html'); do
106-
python $DIR/delete_html_file_with_runnable_code_removed.py $filename
115+
for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.rst'); do
116+
file_basename=$(basename $filename .rst)
117+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
118+
rm $filename
119+
fi
107120
done
108-
for filename in $(find docs/advanced -name '*.html'); do
109-
python $DIR/delete_html_file_with_runnable_code_removed.py $filename
121+
for filename in $(find docs/_downloads -name '*.py'); do
122+
file_basename=$(basename $filename .py)
123+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
124+
rm $filename
125+
fi
126+
done
127+
for filename in $(find docs/_downloads -name '*.ipynb'); do
128+
file_basename=$(basename $filename .ipynb)
129+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
130+
rm $filename
131+
fi
132+
done
133+
for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced -name '*.rst.txt'); do
134+
file_basename=$(basename $filename .rst.txt)
135+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
136+
rm $filename
137+
fi
138+
done
139+
for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced -name '*.doctree'); do
140+
file_basename=$(basename $filename .doctree)
141+
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
142+
rm $filename
143+
fi
110144
done
111145

112-
# Step 5: Remove INVISIBLE_CODE_BLOCK from all HTML files
146+
# Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files
113147
bash $DIR/remove_invisible_code_block_batch.sh docs
114148

115-
# Step 6: Copy generated HTML files and static files to S3, tag with commit ID
149+
# Step 6: Copy generated files to S3, tag with commit ID
116150
7z a worker_${WORKER_ID}.7z docs
117151
aws s3 cp worker_${WORKER_ID}.7z s3://${BUCKET_NAME}/${COMMIT_ID}/worker_${WORKER_ID}.7z --acl public-read
118152
elif [[ "${JOB_BASE_NAME}" == *manager ]]; then
@@ -151,14 +185,17 @@ elif [[ "${JOB_BASE_NAME}" == *manager ]]; then
151185
done
152186

153187
# Step 5: Copy all static files into docs
154-
rsync -av docs_with_plot/docs/ docs --exclude beginner --exclude intermediate --exclude advanced
188+
rsync -av docs_with_plot/docs/ docs
189+
190+
# Step 6: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files
191+
bash $DIR/remove_invisible_code_block_batch.sh docs
155192

156-
# Step 6: Copy generated HTML files and static files to S3
193+
# Step 7: Copy generated HTML files and static files to S3
157194
7z a manager.7z docs
158195
aws s3 cp manager.7z s3://${BUCKET_NAME}/${COMMIT_ID}/manager.7z --acl public-read
159196

160197
# yf225 TODO: re-enable this after tutorial build is fixed
161-
# # Step 7: push new HTML files and static files to gh-pages
198+
# # Step 8: push new HTML files and static files to gh-pages
162199
# if [[ "$COMMIT_SOURCE" == master ]]; then
163200
# git clone https://github.com/pytorch/tutorials.git -b gh-pages gh-pages
164201
# cp -r docs/* gh-pages/

‎.jenkins/remove_invisible_code_block_batch.sh

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,20 @@ BUILDDIR=$1
22

33
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
44

5-
# Remove INVISIBLE_CODE_BLOCK from all HTML files
6-
for filename in $(find $BUILDDIR/beginner -name '*.html'); do
5+
# Remove INVISIBLE_CODE_BLOCK from .html/.rst/.rst.txt/.ipynb/.py files
6+
for filename in $(find $BUILDDIR/beginner $BUILDDIR/intermediate $BUILDDIR/advanced -name '*.html'); do
77
echo "Removing INVISIBLE_CODE_BLOCK from " $filename
88
python $DIR/remove_invisible_code_block_from_html.py $filename $filename
99
done
10-
for filename in $(find $BUILDDIR/intermediate -name '*.html'); do
10+
for filename in $(find $BUILDDIR/_sources/beginner $BUILDDIR/_sources/intermediate $BUILDDIR/_sources/advanced -name '*.rst.txt'); do
1111
echo "Removing INVISIBLE_CODE_BLOCK from " $filename
12-
python $DIR/remove_invisible_code_block_from_html.py $filename $filename
12+
python $DIR/remove_invisible_code_block_from_rst_txt.py $filename $filename
1313
done
14-
for filename in $(find $BUILDDIR/advanced -name '*.html'); do
14+
for filename in $(find $BUILDDIR/_downloads -name '*.ipynb'); do
1515
echo "Removing INVISIBLE_CODE_BLOCK from " $filename
16-
python $DIR/remove_invisible_code_block_from_html.py $filename $filename
17-
done
16+
python $DIR/remove_invisible_code_block_from_ipynb.py $filename $filename
17+
done
18+
for filename in $(find $BUILDDIR/_downloads -name '*.py'); do
19+
echo "Removing INVISIBLE_CODE_BLOCK from " $filename
20+
python $DIR/remove_invisible_code_block_from_py.py $filename $filename
21+
done

‎.jenkins/remove_invisible_code_block_from_html.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
output_file_path = sys.argv[2]
66

77
with open(html_file_path, 'r', encoding='utf-8') as html_file:
8-
html = html_file.read()
8+
html = html_file.read()
99
html_soup = BeautifulSoup(html, 'html.parser')
1010

1111
elems = html_soup.find_all("div", {"class": "highlight-python"})

‎.jenkins/remove_invisible_code_block_from_ipynb.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import sys
2+
from bs4 import BeautifulSoup
3+
4+
ipynb_file_path = sys.argv[1]
5+
output_file_path = sys.argv[2]
6+
7+
with open(ipynb_file_path, 'r', encoding='utf-8') as ipynb_file:
8+
ipynb_lines = ipynb_file.readlines()
9+
10+
ipynb_out_lines = []
11+
12+
for line in ipynb_lines:
13+
if not '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line:
14+
ipynb_out_lines.append(line)
15+
16+
with open(output_file_path, "w", encoding='utf-8') as output_file:
17+
for line in ipynb_out_lines:
18+
output_file.write(line)

‎.jenkins/remove_invisible_code_block_from_py.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import sys
2+
from bs4 import BeautifulSoup
3+
4+
py_file_path = sys.argv[1]
5+
output_file_path = sys.argv[2]
6+
7+
with open(py_file_path, 'r', encoding='utf-8') as py_file:
8+
py_lines = py_file.readlines()
9+
10+
py_out_lines = []
11+
12+
in_invisible_block = False
13+
for line in py_lines:
14+
if not in_invisible_block:
15+
if '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line:
16+
in_invisible_block = True
17+
else:
18+
py_out_lines.append(line)
19+
else:
20+
if '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line:
21+
in_invisible_block = False
22+
23+
with open(output_file_path, "w", encoding='utf-8') as output_file:
24+
for line in py_out_lines:
25+
output_file.write(line)

‎.jenkins/remove_invisible_code_block_from_rst_txt.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import sys
2+
from bs4 import BeautifulSoup
3+
4+
rst_txt_file_path = sys.argv[1]
5+
output_file_path = sys.argv[2]
6+
7+
with open(rst_txt_file_path, 'r', encoding='utf-8') as rst_txt_file:
8+
rst_txt = rst_txt_file.read()
9+
10+
splits = rst_txt.split('.. code-block:: python\n\n\n # %%%%%%INVISIBLE_CODE_BLOCK%%%%%%\n')
11+
if len(splits) == 2:
12+
code_before_invisible_block = splits[0]
13+
code_after_invisible_block = splits[1].split(' # %%%%%%INVISIBLE_CODE_BLOCK%%%%%%\n')[1]
14+
rst_txt_out = code_before_invisible_block + code_after_invisible_block
15+
else:
16+
rst_txt_out = rst_txt
17+
18+
with open(output_file_path, "w", encoding='utf-8') as output_file:
19+
output_file.write(rst_txt_out)

‎beginner_source/former_torchies/README.txt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
PyTorch for former Torch users
22
------------------------------
33

4-
1. tensor_tutorial.py
4+
1. tensor_tutorial_old.py
55
Tensors
6-
https://pytorch.org/tutorials/beginner/former_torchies/tensor_tutorial.html
6+
https://pytorch.org/tutorials/beginner/former_torchies/tensor_tutorial_old.html
77

8-
2. autograd.py
8+
2. autograd_tutorial_old.py
99
Autograd
10-
https://pytorch.org/tutorials/beginner/former_torchies/autograd_tutorial.html
10+
https://pytorch.org/tutorials/beginner/former_torchies/autograd_tutorial_old.html
1111

1212
3. nn_tutorial.py
1313
nn package

‎beginner_source/former_torchies/autograd_tutorial.py renamed to ‎beginner_source/former_torchies/autograd_tutorial_old.py

File renamed without changes.

‎beginner_source/former_torchies/tensor_tutorial.py renamed to ‎beginner_source/former_torchies/tensor_tutorial_old.py

File renamed without changes.

‎beginner_source/former_torchies_tutorial.rst

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@ In this tutorial, you will learn the following:
1717
.. toctree::
1818
:hidden:
1919

20-
/beginner/former_torchies/tensor_tutorial
21-
/beginner/former_torchies/autograd_tutorial
20+
/beginner/former_torchies/tensor_tutorial_old
21+
/beginner/former_torchies/autograd_tutorial_old
2222
/beginner/former_torchies/nn_tutorial
2323
/beginner/former_torchies/parallelism_tutorial
2424

25-
.. galleryitem:: /beginner/former_torchies/tensor_tutorial.py
25+
.. galleryitem:: /beginner/former_torchies/tensor_tutorial_old.py
2626
:figure: /_static/img/tensor_illustration_flat.png
2727

28-
.. galleryitem:: /beginner/former_torchies/autograd_tutorial.py
28+
.. galleryitem:: /beginner/former_torchies/autograd_tutorial_old.py
2929

3030
.. galleryitem:: /beginner/former_torchies/nn_tutorial.py
3131
:figure: /_static/img/torch-nn-vs-pytorch-nn.png

0 commit comments

Comments
 (0)