Skip to content

Commit

Permalink
Fix/javadoc script (#555)
Browse files Browse the repository at this point in the history
  • Loading branch information
karllessard committed Sep 24, 2024
1 parent 9cf8693 commit 889d74a
Show file tree
Hide file tree
Showing 4 changed files with 150 additions and 34 deletions.
47 changes: 23 additions & 24 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,28 @@ jobs:
fi
echo "Repository URL: $REPOSITORY_URL"
echo "::set-output name=repositoryUrl::$REPOSITORY_URL"
# linux-arm64:
# runs-on: linux-arm64-ubuntu2204
# needs: prepare
# strategy:
# matrix:
# ext: [""]
# steps:
# - name: Install environment
# run: |
# sudo apt update
# sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
# - name: Checkout repository
# uses: actions/checkout@v1
# - name: Build project
# run: |
# gcc --version
# mvn -version
# echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
# mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
# - name: Deploy native artifact
# if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
# run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
linux-x86_64:
runs-on: ubuntu-20.04
needs: prepare
Expand Down Expand Up @@ -178,32 +200,9 @@ jobs:
run: |
call mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}
if ERRORLEVEL 1 exit /b
linux-arm64:
runs-on: linux-arm64-ubuntu2204
needs: prepare
strategy:
matrix:
ext: [""]
steps:
- name: Install environment
run: |
sudo apt update
sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk
- name: Checkout repository
uses: actions/checkout@v1
- name: Build project
run: |
gcc --version
mvn -version
echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }}
- name: Deploy native artifact
if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }}

deploy:
if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging') }} # DEPLOY_SNAPSHOT (releases should be signed and deployed manually from local machine)
needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64, linux-arm64]
needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64] #, linux-arm64]
runs-on: ubuntu-20.04
steps:
- name: Configure Java
Expand Down
1 change: 1 addition & 0 deletions tensorflow-core/tensorflow-core-native/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,7 @@
<minmemory>256m</minmemory>
<maxmemory>2048m</maxmemory>
<links>
<link>https://protobuf.dev/reference/java/api-docs</link>
<link>http://bytedeco.org/javacpp/apidocs</link>
</links>
</configuration>
Expand Down
38 changes: 28 additions & 10 deletions tools/build_java_api_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate TensorFlow Lite Java reference docs for TensorFlow.org."""
"""Generate TensorFlow Java reference docs for TensorFlow.org."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pathlib
import shutil
import tempfile
from git import Repo

from absl import app
from absl import flags

from tensorflow_docs.api_generator import gen_java

FLAGS = flags.FLAGS
NDARRAY_VERSION = 'v1.0.0'

# These flags are required by infrastructure, not all of them are used.
flags.DEFINE_string('output_dir', '/tmp/java_api/',
Expand All @@ -48,6 +50,18 @@
TOOLS_DIR = pathlib.Path(__file__).resolve().parent
REPO_ROOT = TOOLS_DIR.parent


def checkout_ndarray():
repo_url = 'https://github.com/tensorflow/java-ndarray'
local_repo_path = REPO_ROOT/'ndarray'
if not pathlib.Path(local_repo_path).exists():
local_repo = Repo.clone_from(repo_url, local_repo_path)
else:
local_repo = Repo(local_repo_path)
local_repo.remotes['origin'].fetch()
local_repo.git.checkout(NDARRAY_VERSION)


def overlay(from_root, to_root):
for from_path in pathlib.Path(from_root).rglob('*'):
relpath = from_path.relative_to(from_root)
Expand All @@ -58,24 +72,28 @@ def overlay(from_root, to_root):
else:
to_path.mkdir(exist_ok=True)


def main(unused_argv):
checkout_ndarray()
merged_source = pathlib.Path(tempfile.mkdtemp())
(merged_source / 'java/org').mkdir(parents=True)

shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/',
merged_source/'java/org/tensorflow')
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow',
merged_source/'java/org/tensorflow')
shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework',
merged_source/'java/org/tensorflow/framework')
shutil.copytree(REPO_ROOT/'ndarray/src/main/java/org/tensorflow/ndarray',
merged_source/'java/org/tensorflow/ndarray')
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow')
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', merged_source/'java/org/tensorflow')
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/proto', merged_source/'java/org/tensorflow/proto')
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/exceptions', merged_source/'java/org/tensorflow/exceptions')
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api', merged_source/'java/org/tensorflow/internal/c_api')
shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', merged_source/'java/org/tensorflow/framework')
shutil.copytree(REPO_ROOT/'ndarray/ndarray/src/main/java/org/tensorflow/ndarray', merged_source/'java/org/tensorflow/ndarray')

gen_java.gen_java_docs(
package='org.tensorflow',
source_path=merged_source / 'java',
output_dir=pathlib.Path(FLAGS.output_dir),
site_path=pathlib.Path(FLAGS.site_path))
site_path=pathlib.Path(FLAGS.site_path),
# Uncomment for local testing:
# script_path=pathlib.Path(REPO_ROOT/'tools/run-javadoc-for-tf-local.sh'),
)


if __name__ == '__main__':
Expand Down
98 changes: 98 additions & 0 deletions tools/run-javadoc-for-tf-local.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/bin/bash
set -ex

export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home # Or change to any JDK 11 home path

# https://android.googlesource.com/platform/external/doclava/
# There's a debian package:
# https://packages.debian.org/unstable/doclava-aosp
# Install with:
#
# $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1
#
# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned
DOCLAVA_JAR=${DOCLAVA_JAR:-'lib/doclava.jar'} # Build lib locally

# Install java clear silver templates with:
#
# $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1
JSILVER_JAR=${JSILVER_JAR:-'lib/jsilver.jar'} # Build lib locally


######### DELETE OUTPUT_DIR #################

# Empty the output directory in case a class has been deleted
rm -rf "${OUTPUT_DIR:?}"/*
############ RUN DOCLAVA ###################

# $FEDERATED_DOCS is a space-separated string of url,file pairs.
read -a api_pairs <<< "${FEDERATED_DOCS}"
FEDERATED_PARAMS=""
for i in "${!api_pairs[@]}"; do
api_pair_str="${api_pairs[$i]}" # "url,api.txt"
read -a api_pair <<< "${api_pair_str//,/ }"
# Using the index as the API "name", build the federation params. Note that
# using 0 as an API name will evaluate to false and cause rendering bugs,
# so we preface with "api_".
FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}"
FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}"
done

# To install javadoc, for example, use
#
# sudo apt install openjdk-11-jdk
#
# doclava doesn't work with openjdk-13
# ```
# javadoc: error - Class com.google.doclava.Doclava is not a valid doclet.
# Note: As of JDK 13, the com.sun.javadoc API is no longer supported.
# ```
# It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java

# Each package in $PACKAGE needs to prefaced with -subpackages, so do that.
SUBPACKAGES=""
read -r -a packages <<< "${PACKAGE}"
for pkg in "${packages[@]}"; do
SUBPACKAGES+=" -subpackages ${pkg}"
done
( # Capture the return code. it may be non-zero for minor errors.
javadoc \
-sourcepath "${SOURCE_PATH}" \
-docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \
-doclet com.google.doclava.Doclava \
-d "${OUTPUT_DIR}" \
-toroot "${SITE_PATH}"/ \
-yaml _toc.yaml \
-templatedir "${TEMPLATES}" \
-public \
-devsite \
${FEDERATED_PARAMS} \
${SUBPACKAGES}
)


mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}"

###################################################################
################### START OF POST-PROCESSING ######################
###################################################################
rm "${OUTPUT_DIR}/navtree_data.js" || true
rm "${OUTPUT_DIR}/hierarchy.html" || true

find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
find ${OUTPUT_DIR} -name "*.yaml" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g"
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g"

JAVA_LANG=https://docs.oracle.com/javase/8/docs/api
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g"

find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' 's|<pre><code>|<pre class="prettyprint"><code>|g'

rm ${OUTPUT_DIR}/timestamp.js || true
rm ${OUTPUT_DIR}/lists.js || true
rm ${OUTPUT_DIR}/index.html || true

cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/


0 comments on commit 889d74a

Please sign in to comment.