From d8ba114bb94275d3fee0a4ad98d8516ef90f4bdf Mon Sep 17 00:00:00 2001 From: Karl Lessard Date: Sat, 31 Aug 2024 13:02:45 -0400 Subject: [PATCH] Fix/javadoc script (#555) --- .github/workflows/build.yml | 47 +++++---- .../tensorflow-core-native/pom.xml | 1 + tools/build_java_api_docs.py | 38 +++++-- tools/run-javadoc-for-tf-local.sh | 98 +++++++++++++++++++ 4 files changed, 150 insertions(+), 34 deletions(-) create mode 100644 tools/run-javadoc-for-tf-local.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54bef3a9604..4bd0d04bb30 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,6 +62,28 @@ jobs: fi echo "Repository URL: $REPOSITORY_URL" echo "::set-output name=repositoryUrl::$REPOSITORY_URL" +# linux-arm64: +# runs-on: linux-arm64-ubuntu2204 +# needs: prepare +# strategy: +# matrix: +# ext: [""] +# steps: +# - name: Install environment +# run: | +# sudo apt update +# sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk +# - name: Checkout repository +# uses: actions/checkout@v1 +# - name: Build project +# run: | +# gcc --version +# mvn -version +# echo "ossrh${{ secrets.CI_DEPLOY_USERNAME }}${{ secrets.CI_DEPLOY_PASSWORD }}" > $HOME/.m2/settings.xml +# mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} +# - name: Deploy native artifact +# if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true' +# run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }} linux-x86_64: runs-on: ubuntu-20.04 needs: prepare @@ -178,32 +200,9 @@ jobs: run: | call mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }} if ERRORLEVEL 1 exit /b - linux-arm64: - runs-on: linux-arm64-ubuntu2204 - needs: prepare - strategy: - matrix: - ext: [""] - steps: - - name: Install environment - run: | - sudo apt update - sudo apt install -y curl wget unzip tar git gcc g++ maven default-jdk - - name: Checkout repository - uses: actions/checkout@v1 - - name: Build project - run: | - gcc --version - mvn -version - echo "ossrh${{ secrets.CI_DEPLOY_USERNAME }}${{ secrets.CI_DEPLOY_PASSWORD }}" > $HOME/.m2/settings.xml - mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} - - name: Deploy native artifact - if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true' - run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -e -Djavacpp.platform=${{ github.job }} -Djavacpp.platform.extension=${{ matrix.ext }} -Durl=${{ needs.prepare.outputs.repositoryUrl }} - deploy: if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging') }} # DEPLOY_SNAPSHOT (releases should be signed and deployed manually from local machine) - needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64, linux-arm64] + needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64] #, linux-arm64] runs-on: ubuntu-20.04 steps: - name: Configure Java diff --git a/tensorflow-core/tensorflow-core-native/pom.xml b/tensorflow-core/tensorflow-core-native/pom.xml index 992c75df82f..23682984a4a 100644 --- a/tensorflow-core/tensorflow-core-native/pom.xml +++ b/tensorflow-core/tensorflow-core-native/pom.xml @@ -669,6 +669,7 @@ 256m 2048m + https://protobuf.dev/reference/java/api-docs http://bytedeco.org/javacpp/apidocs diff --git a/tools/build_java_api_docs.py b/tools/build_java_api_docs.py index 42a83703ecf..5eadafc276d 100644 --- a/tools/build_java_api_docs.py +++ b/tools/build_java_api_docs.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Generate TensorFlow Lite Java reference docs for TensorFlow.org.""" +"""Generate TensorFlow Java reference docs for TensorFlow.org.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -21,6 +21,7 @@ import pathlib import shutil import tempfile +from git import Repo from absl import app from absl import flags @@ -28,6 +29,7 @@ from tensorflow_docs.api_generator import gen_java FLAGS = flags.FLAGS +NDARRAY_VERSION = 'v1.0.0' # These flags are required by infrastructure, not all of them are used. flags.DEFINE_string('output_dir', '/tmp/java_api/', @@ -48,6 +50,18 @@ TOOLS_DIR = pathlib.Path(__file__).resolve().parent REPO_ROOT = TOOLS_DIR.parent + +def checkout_ndarray(): + repo_url = 'https://github.com/tensorflow/java-ndarray' + local_repo_path = REPO_ROOT/'ndarray' + if not pathlib.Path(local_repo_path).exists(): + local_repo = Repo.clone_from(repo_url, local_repo_path) + else: + local_repo = Repo(local_repo_path) + local_repo.remotes['origin'].fetch() + local_repo.git.checkout(NDARRAY_VERSION) + + def overlay(from_root, to_root): for from_path in pathlib.Path(from_root).rglob('*'): relpath = from_path.relative_to(from_root) @@ -58,24 +72,28 @@ def overlay(from_root, to_root): else: to_path.mkdir(exist_ok=True) + def main(unused_argv): + checkout_ndarray() merged_source = pathlib.Path(tempfile.mkdtemp()) (merged_source / 'java/org').mkdir(parents=True) - shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', - merged_source/'java/org/tensorflow') - overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', - merged_source/'java/org/tensorflow') - shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', - merged_source/'java/org/tensorflow/framework') - shutil.copytree(REPO_ROOT/'ndarray/src/main/java/org/tensorflow/ndarray', - merged_source/'java/org/tensorflow/ndarray') + shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow') + overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', merged_source/'java/org/tensorflow') + shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/proto', merged_source/'java/org/tensorflow/proto') + shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/exceptions', merged_source/'java/org/tensorflow/exceptions') + shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api', merged_source/'java/org/tensorflow/internal/c_api') + shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', merged_source/'java/org/tensorflow/framework') + shutil.copytree(REPO_ROOT/'ndarray/ndarray/src/main/java/org/tensorflow/ndarray', merged_source/'java/org/tensorflow/ndarray') gen_java.gen_java_docs( package='org.tensorflow', source_path=merged_source / 'java', output_dir=pathlib.Path(FLAGS.output_dir), - site_path=pathlib.Path(FLAGS.site_path)) + site_path=pathlib.Path(FLAGS.site_path), + # Uncomment for local testing: + # script_path=pathlib.Path(REPO_ROOT/'tools/run-javadoc-for-tf-local.sh'), + ) if __name__ == '__main__': diff --git a/tools/run-javadoc-for-tf-local.sh b/tools/run-javadoc-for-tf-local.sh new file mode 100644 index 00000000000..59239b78141 --- /dev/null +++ b/tools/run-javadoc-for-tf-local.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -ex + +export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home # Or change to any JDK 11 home path + +# https://android.googlesource.com/platform/external/doclava/ +# There's a debian package: +# https://packages.debian.org/unstable/doclava-aosp +# Install with: +# +# $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1 +# +# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned +DOCLAVA_JAR=${DOCLAVA_JAR:-'lib/doclava.jar'} # Build lib locally + +# Install java clear silver templates with: +# +# $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1 +JSILVER_JAR=${JSILVER_JAR:-'lib/jsilver.jar'} # Build lib locally + + +######### DELETE OUTPUT_DIR ################# + +# Empty the output directory in case a class has been deleted +rm -rf "${OUTPUT_DIR:?}"/* +############ RUN DOCLAVA ################### + +# $FEDERATED_DOCS is a space-separated string of url,file pairs. +read -a api_pairs <<< "${FEDERATED_DOCS}" +FEDERATED_PARAMS="" +for i in "${!api_pairs[@]}"; do + api_pair_str="${api_pairs[$i]}" # "url,api.txt" + read -a api_pair <<< "${api_pair_str//,/ }" + # Using the index as the API "name", build the federation params. Note that + # using 0 as an API name will evaluate to false and cause rendering bugs, + # so we preface with "api_". + FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}" + FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}" +done + +# To install javadoc, for example, use +# +# sudo apt install openjdk-11-jdk +# +# doclava doesn't work with openjdk-13 +# ``` +# javadoc: error - Class com.google.doclava.Doclava is not a valid doclet. +# Note: As of JDK 13, the com.sun.javadoc API is no longer supported. +# ``` +# It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java + +# Each package in $PACKAGE needs to prefaced with -subpackages, so do that. +SUBPACKAGES="" +read -r -a packages <<< "${PACKAGE}" +for pkg in "${packages[@]}"; do + SUBPACKAGES+=" -subpackages ${pkg}" +done +( # Capture the return code. it may be non-zero for minor errors. + javadoc \ + -sourcepath "${SOURCE_PATH}" \ + -docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \ + -doclet com.google.doclava.Doclava \ + -d "${OUTPUT_DIR}" \ + -toroot "${SITE_PATH}"/ \ + -yaml _toc.yaml \ + -templatedir "${TEMPLATES}" \ + -public \ + -devsite \ + ${FEDERATED_PARAMS} \ + ${SUBPACKAGES} +) + + +mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}" + +################################################################### +################### START OF POST-PROCESSING ###################### +################################################################### +rm "${OUTPUT_DIR}/navtree_data.js" || true +rm "${OUTPUT_DIR}/hierarchy.html" || true + +find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g" +find ${OUTPUT_DIR} -name "*.yaml" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g" +find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g" +find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g" + +JAVA_LANG=https://docs.oracle.com/javase/8/docs/api +find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g" + +find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' 's|
|
|g'
+
+rm ${OUTPUT_DIR}/timestamp.js || true
+rm ${OUTPUT_DIR}/lists.js || true
+rm ${OUTPUT_DIR}/index.html || true
+
+cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/
+
+