Docs build all versions refactor (apache#9878)

* changed url references from dmlc to apache/incubator-mxnet

* refactored build all docs

* Created dockerfile and modified scripts to support it.

* add license header

* used license header tool this time

* updated scripts to take arguments for flexible use

docs/build_version_doc/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:16.04
+LABEL maintainer="markhama@amazon.com"
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    apt-transport-https \
+    build-essential \
+    ca-certificates \
+    curl \
+    doxygen \
+    git \
+    libatlas-base-dev \
+    liblapack-dev \
+    libopenblas-dev \
+    libopencv-dev \
+    pandoc \
+    python-numpy \
+    python-pip \
+    software-properties-common \
+    unzip \
+    wget
+
+# Setup Scala
+RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+RUN apt-get update && apt-get install -y \
+    sbt \
+    scala 
+
+RUN pip install --upgrade pip && pip install \
+    beautifulsoup4 \
+    breathe \
+    CommonMark==0.5.4 \
+    h5py \
+    mock==1.0.1 \
+    pypandoc \
+    recommonmark==0.4.0 \
+    sphinx==1.5.6 
+
+
+COPY *.sh /
+COPY *.py /
+RUN /build_all_version.sh "1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master"
+RUN /update_all_version.sh "1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master" 1.1.0 http://mxnet.incubator.apache.org/

docs/build_version_doc/build_all_version.sh

@@ -19,67 +19,65 @@
 
 # This script is for locally building website for all versions
 # Built files are stored in $built
-# Version numbers are stored in $tag_list.
-# Version numbers are ordered from latest to old and final one is master.
+
+# Takes one argument:
+# * tag list - space delimited list of Github tags; Example: "1.1.0 1.0.0 master"
+# Example Usage:
+# ./build_all_version.sh "1.1.0 1.0.0 master"
+
 set -e
 set -x
 
-tag_list="1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master"
+if [ -z "$1" ]
+  then
+    echo "Please provide a list of version tags you wish to run."
+    exit 1
+  else
+    tag_list="$1"
+    echo "Using these tags: $1"
+fi
 
 mxnet_url="https://github.com/apache/incubator-mxnet.git"
 mxnet_folder="apache_mxnet"
 built="VersionedWeb"
-mkdir $built
-mkdir "$built/versions"
 
-git clone $mxnet_url $mxnet_folder --recursive
-cd "$mxnet_folder/docs"
-tag_file="tag_list.txt"
+if [ ! -d "$mxnet_folder" ]; then
+  mkdir $mxnet_folder
+  git clone $mxnet_url $mxnet_folder --recursive
+fi
 
-# Write all version numbers into $tag_file
-for tag in $tag_list; do
-    if [ $tag != 'master' ]
-    then
-        echo "$tag" >> "$tag_file"
-    fi
-done
+if [ ! -d "$built" ]; then
+  mkdir $built
+  mkdir "$built/versions"
+fi
 
 # Build all versions and use latest version(First version number in $tag_list) as landing page.
-version_num=0
 for tag in $tag_list; do
+    cd "$mxnet_folder"
+    git fetch
     if [ $tag == 'master' ]
-    then
-        git checkout master
-    else
-        git checkout "tags/$tag"
+        then
+            git checkout master
+            git pull
+        else
+            git checkout "tags/$tag"
+    fi
+    if [ $tag == '0.11.0' ]
+      then
+          git checkout master -- docs/mxdoc.py
     fi
-
     git submodule update || exit 1
-    cd ..
     make clean
     cd docs
     make clean
-    make html USE_OPENMP=0 || exit 1
-    python build_version_doc/AddVersion.py --file_path "_build/html/" --current_version "$tag" || exit 1
-
-    if [ $tag != 'master' ]
-    then 
-        python build_version_doc/AddPackageLink.py --file_path "_build/html/get_started/install.html" \
-                                                   --current_version "$tag" || exit 1
-    fi
-
-    if [ $version_num == 0 ]
-    then
-        cp -a _build/html/. "../../$built"
-    else
-        file_loc="../../$built/versions/$tag"
-        mkdir "$file_loc"
-        cp -a _build/html/. "$file_loc"
+    make html USE_OPENMP=1 || exit 1
+    cd ../../
+    file_loc="$built/versions/$tag"
+    if [ -d "$file_loc" ] ; then
+        rm -rf "$file_loc"
     fi
-
-    ((++version_num))
+    mkdir "$file_loc"
+    cp -a "$mxnet_folder/docs/_build/html/." "$file_loc"
 done
-
-mv "$tag_file" "../../$built/tag.txt"
-cd ../..
-rm -rf "$mxnet_folder"
+
+echo "Now you may want to run update_all_version.sh to create the production layout with the versions dropdown and other per-version corrections."

docs/build_version_doc/build_site_tag.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# How this script works:
+# 1. Receive tag list
+#    Looks like: tag_list="1.1.0 1.0.0 0.12.1 0.12.0 0.11.0 master"
+# 2. Receive default tag (for main website view)
+# 3. Receive root URL
+# 4. Call build and then update scripts
+
+# Take user input or check env var for tag list
+if [ -z "$1" ]
+  then
+    echo "No tag list supplied... trying environment variable $TAG_LIST"
+  else
+    tag_list="${TAG_LIST:-"$1"}"
+    echo "Using these tags: $1"
+fi
+
+if [ -z "$tag_list" ]
+  then
+    echo "No tags defined"
+    exit 1
+fi
+
+if [ -z "$2" ]
+  then
+    echo "Please pick a version to use as a default for the website. Ex: 1.1.0"
+    exit 1
+  else
+    tag_default=$2
+fi
+
+if [ -z "$3" ]
+  then
+    echo "Please provide the root url for the site. Ex: http://mxnet.incubator.apache.org/"
+    exit 1
+  else
+    root_url=$3
+fi
+
+# Pass params to build and update scripts
+for tag in $tag_list; do
+  ./build_all_version.sh $tag || exit 1
+done
+
+./update_all_version.sh "$tag_list" $tag_default $root_url || exit 1
+

docs/build_version_doc/setup_docs_ubuntu.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# If you need to build <= v0.12.0 then use a Python 2 environment
+# mxdoc.py - a sphinx extension, was not Python 3 compatible in the old versions
+# source activate mxnet_p27
+
+# Install dependencies
+sudo apt-get update
+sudo apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    doxygen \
+    software-properties-common
+
+pip install --user \
+    beautifulsoup4 \
+    breathe \
+    CommonMark==0.5.4 \
+    h5py \
+    mock==1.0.1 \
+    pypandoc \
+    recommonmark==0.4.0 \
+    sphinx==1.5.6
+
+# Recommonmark/Sphinx errors: https://github.com/sphinx-doc/sphinx/issues/3800
+
+
+# Setup scala
+echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+sudo apt-get update
+sudo apt-get install -y \
+  sbt \
+  scala
+
+# Cleanup
+sudo apt autoremove -y
+
+# Make docs using the manual way
+# cd .. && make html USE_OPENMP=0
+# using the docker way
+# sudo make docs
+

docs/build_version_doc/update_all_version.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script will update the html content from building 
+# different tags.
+# It assumes you have already run build_all_version.sh for 
+# the tags you want to update.
+
+# Takes three arguments:
+# * tag list - space delimited list of Github tags; Example: "1.1.0 1.0.0 master"
+# * default tag - which version should the site default to; Example: 1.0.0
+# * root URL - for the versions dropdown to change to production or dev server; Example: http://mxnet.incubator.apache.org/
+
+# Example Usage:
+# ./update_all_version.sh "1.1.0 1.0.0 master" 1.0.0 http://mxnet.incubator.apache.org/
+
+set -e
+set -x
+
+if [ -z "$1" ]
+  then    
+    echo "Please provide a list of version tags you wish to run. Ex : \"1.1.0 1.0.0 master\""
+    exit 1
+  else
+    tag_list=$1
+fi    
+
+if [ -z "$2" ]
+  then    
+    echo "Please pick a version to use as a default for the website. Ex: 1.1.0"
+    exit 1
+  else
+    tag_default=$2
+fi    
+
+if [ -z "$3" ]
+  then
+    echo "Please provide the root url for the site. Ex: http://mxnet.incubator.apache.org/"
+    exit 1
+  else
+    root_url=$3
+fi
+
+mxnet_folder="apache_mxnet"
+built="VersionedWeb"
+tag_file="tag_list.txt"
+
+if [ -f "$tag_file" ]; then
+  rm $tag_file
+fi
+
+# Write all version numbers into $tag_file for AddVersion.py to use later
+# Master is added by that script by default
+for tag in $tag_list; do
+    if [ $tag != 'master' ]
+    then
+        echo "$tag" >> "$tag_file"
+    fi
+done
+
+# Update the specified tags with the Versions dropdown
+for tag in $tag_list; do
+    # This Python script is expecting the tag_list.txt and it will use that as the entries to populate
+    python AddVersion.py --root_url "$root_url" --file_path "$built/versions/$tag" --current_version "$tag" || exit 1
+
+    if [ $tag != 'master' ]
+    then 
+        python AddPackageLink.py --file_path "$built/versions/master/install/index.html" \
+                                                   --current_version "$tag" || exit 1
+    fi
+
+    if [ $tag == $tag_default ]
+    then
+        cp -a "$built/versions/$tag/." "$built"
+    else
+        file_loc="$built/versions/$tag"
+        #rm -rf "$file_loc"
+        #mkdir "$file_loc"
+        #cp -a $mxnet_folder/docs/_build/html/. "$file_loc"
+    fi
+done
+
+echo "The output of this process can be found in the VersionedWeb folder."
+