diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e001606a23..1b52d32dffc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,58 +17,66 @@ env: jobs: quick-build: if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI build') - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 container: centos:7 steps: - - name: Checkout repository - uses: actions/checkout@v1 - name: Install environment run: | yum -y update yum -y install centos-release-scl-rh epel-release - yum -y install java-11-openjdk-devel devtoolset-7 + yum -y install devtoolset-9 echo Downloading Maven curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '17' + - name: Checkout repository + uses: actions/checkout@v1 - name: Build project run: | - source scl_source enable devtoolset-7 || true - export JAVA_HOME=$(dirname $(dirname $(readlink $(readlink $(which javac))))) + source scl_source enable devtoolset-9 || true echo $JAVA_HOME mvn -version - mvn clean install -Pdev,jdk11 -B -U -e -Dlint.skip=true + mvn clean install -Pdev,jdk17 -B -U -e -Dlint.skip=true - name: Run lint checks run: | - mvn compiler:compile -Pdev,jdk11 -B -U -e + mvn compiler:compile -Pdev,jdk17 -B -U -e check-format: if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 container: centos:7 steps: - - name: Checkout repository - uses: actions/checkout@v1 - name: Install environment run: | yum -y update yum -y install centos-release-scl-rh epel-release - yum -y install java-11-openjdk-devel devtoolset-7 + yum -y install devtoolset-9 echo Downloading Maven curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '17' + - name: Checkout repository + uses: actions/checkout@v1 - name: Build project run: | - source scl_source enable devtoolset-7 || true - export JAVA_HOME=$(dirname $(dirname $(readlink $(readlink $(which javac))))) + source scl_source enable devtoolset-9 || true echo $JAVA_HOME mvn -version - mvn clean install -Pdev,jdk11 -B -U -e -Dlint.skip=true -Dmaven.test.skip=true + mvn clean install -Pdev,jdk17 -B -U -e -Dlint.skip=true -Dmaven.test.skip=true - name: Run format checks run: | - mvn spotless:check -Pdev,jdk11 -B -U -e + mvn spotless:check -Pdev,jdk17 -B -U -e prepare: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 outputs: stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }} steps: @@ -85,7 +93,7 @@ jobs: echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID" linux-x86_64: if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 container: centos:7 needs: prepare strategy: @@ -99,13 +107,13 @@ jobs: yum --disablerepo updates -y install $GLIBC yum -x "$GLIBC" -y update yum -x "$GLIBC" -y install centos-release-scl-rh epel-release - yum -x "$GLIBC" -y install java-1.8.0-openjdk-devel devtoolset-7 rh-git218 patch perl-Data-Dumper python36-devel python36-numpy python36-pip python36-six + yum -x "$GLIBC" -y install devtoolset-9 rh-git218 patch perl-Data-Dumper python36-devel python36-numpy python36-pip python36-six echo Downloading Maven curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn echo Downloading Bazel - curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-linux-x86_64.sh -o bazel.sh --retry 10 + curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-linux-x86_64.sh -o bazel.sh --retry 10 bash bazel.sh if [[ "${{ matrix.ext }}" == *-gpu ]]; then echo Installing CUDA @@ -124,11 +132,16 @@ jobs: rm -f $(find /usr/local/cuda/ -name '*.a' -and -not -name libcudart_static.a -and -not -name libcudadevrt.a) rm -rf /usr/local/cuda/doc* /usr/local/cuda/libnvvp* /usr/local/cuda/nsight* /usr/local/cuda/samples* fi + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' - name: Checkout repository uses: actions/checkout@v1 - name: Build project run: | - source scl_source enable devtoolset-7 rh-git218 || true + source scl_source enable devtoolset-9 rh-git218 || true git --version gcc --version mvn -version @@ -150,7 +163,7 @@ jobs: df -h macosx-x86_64: if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') - runs-on: macos-latest + runs-on: macos-10.15 needs: prepare strategy: matrix: @@ -160,9 +173,14 @@ jobs: run: | python3 -m pip install numpy six echo Downloading Bazel - curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-darwin-x86_64.sh -o bazel.sh --retry 10 + curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10 bash bazel.sh brew install libomp perl + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' - name: Checkout repository uses: actions/checkout@v1 - name: Build project @@ -186,7 +204,7 @@ jobs: df -h windows-x86_64: if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') - runs-on: windows-latest + runs-on: windows-2019 needs: prepare strategy: matrix: @@ -203,13 +221,13 @@ jobs: run: | set "PATH=C:\msys64\usr\bin;%PATH%" echo Removing broken stuff from WSL and MSYS2 - rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/python.exe" + rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe" python -m pip install numpy six echo Removing old versions of MSVC that interfere with Bazel bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;" echo Downloading Bazel mkdir C:\bazel - curl.exe -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10 + curl.exe -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10 set "EXT=${{ matrix.ext }}" if "%EXT:~-4%" == "-gpu" ( echo Removing some unused stuff to avoid running out of disk space @@ -223,6 +241,11 @@ jobs: cp.exe -a cuda/include cuda/lib cuda/bin "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2/" ) echo %JAVA_HOME% + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' - name: Checkout repository uses: actions/checkout@v1 - name: Build project @@ -264,8 +287,13 @@ jobs: deploy: if: github.event_name == 'push' && contains(github.ref, 'master') needs: [linux-x86_64, macosx-x86_64, windows-x86_64] - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: + - name: Configure Java + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' - name: Checkout repository uses: actions/checkout@v1 - name: Deploy snapshot artifacts diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 00000000000..8488a4fce61 --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1,10 @@ +--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \ No newline at end of file diff --git a/README.md b/README.md index 5c117740db8..3541cb4ed19 100644 --- a/README.md +++ b/README.md @@ -56,12 +56,12 @@ systems, you should add the following dependencies: org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 linux-x86_64${javacpp.platform.extension} ``` @@ -72,24 +72,24 @@ native dependencies as follows: org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 linux-x86_64${javacpp.platform.extension} org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 macosx-x86_64${javacpp.platform.extension} org.tensorflow tensorflow-core-api - 0.3.3 + 0.4.1 windows-x86_64${javacpp.platform.extension} ``` @@ -102,7 +102,7 @@ artifact includes transitively all the artifacts above as a single dependency: org.tensorflow tensorflow-core-platform${javacpp.platform.extension} - 0.3.3 + 0.4.1 ``` @@ -132,23 +132,25 @@ to add Sonatype OSS repository in your pom.xml, like the following org.tensorflow tensorflow-core-platform - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT ``` -## TensorFlow Version Support +## TensorFlow/Java Version Support -This table shows the mapping between different version of TensorFlow for Java and the core runtime libraries. +This table shows the mapping between TensorFlow, TensorFlow Java and minimum supported Java versions. -| TensorFlow Java Version | TensorFlow Version | -| ------------- | ------------- | -| 0.2.0 | 2.3.1 | -| 0.3.0 | 2.4.1 | -| 0.3.1 | 2.4.1 | -| 0.3.2 | 2.4.1 | -| 0.3.3 | 2.4.1 | -| 0.4.0-SNAPSHOT | 2.6.0 +| TensorFlow Java Version | TensorFlow Version | Minimum Java Version | +| ------------- | ------------- | --------------- | +| 0.2.0 | 2.3.1 | 8 | +| 0.3.0 | 2.4.1 | 8 | +| 0.3.1 | 2.4.1 | 8 | +| 0.3.2 | 2.4.1 | 8 | +| 0.3.3 | 2.4.1 | 8 | +| 0.4.0 | 2.7.0 | 8 | +| 0.4.1 | 2.7.1 | 8 | +| 0.5.0-SNAPSHOT | 2.8.0 | 11 | ## How to Contribute? diff --git a/docs/install.md b/docs/install.md index b102782ed4f..bc55a2127d2 100644 --- a/docs/install.md +++ b/docs/install.md @@ -41,14 +41,8 @@ TensorFlow Java to your project. The easiest one is to add a dependency on the Core API and the native dependencies it requires to run on all supported platforms. -You can also select one of the following extensions instead of the pure CPU -version: - -* `tensorflow-core-platform-mkl`: Support for Intel® MKL-DNN on all platforms -* `tensorflow-core-platform-gpu`: Support for CUDA® on Linux and Windows - platforms -* `tensorflow-core-platform-mkl-gpu`: Support for Intel® MKL-DNN and CUDA® on - Linux platform. +You can also select the `tensorflow-core-platform-gpu` extension instead, which +supports CUDA® on Linux and Windows platforms. In addition, a separate dependency on the `tensorflow-framework` library can be added to benefit from a rich set of utilities for TensorFlow-based machine @@ -64,7 +58,7 @@ For example, org.tensorflow tensorflow-core-platform - 0.3.3 + 0.4.1 ``` @@ -107,7 +101,7 @@ snapshots repository in your `pom.xml`. org.tensorflow tensorflow-core-platform - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT ``` @@ -124,7 +118,7 @@ repositories { } dependencies { - compile group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.3.3' + compile group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.4.1' } ``` @@ -170,7 +164,7 @@ add the TensorFlow dependency to the project's `pom.xml` file: org.tensorflow tensorflow-core-platform - 0.3.3 + 0.4.1 @@ -195,8 +189,8 @@ public class HelloTensorFlow { try (ConcreteFunction dbl = ConcreteFunction.create(HelloTensorFlow::dbl); TInt32 x = TInt32.scalarOf(10); - Tensor dblX = dbl.call(x)) { - System.out.println(x.getInt() + " doubled is " + ((TInt32)dblX).getInt()); + TInt32 dblX = (TInt32)dbl.call(x)) { + System.out.println(x.getInt() + " doubled is " + dblX.getInt()); } } diff --git a/pom.xml b/pom.xml index ed06b80c231..d87daefcb7b 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.tensorflow tensorflow-java - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT pom TensorFlow Java Parent @@ -37,16 +37,17 @@ UTF8 - 1.8 - 1.8 + 11 + 11 + 11 5.6.2 1.21 2.7 - 2.6.0 + 2.10.0 true true true - 2.11.1 + 2.20.2 @@ -174,11 +175,11 @@ - jdk11 + jdk17 - 11 - 11 - 11 + 17 + 17 + 17 @@ -189,11 +190,9 @@ lint - - (1.9,) - !lint.skip + lint.skip !true @@ -205,13 +204,15 @@ 3.8.0 true - true -Xlint:all -XDcompilePolicy=simple -Xplugin:ErrorProne - + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED @@ -220,8 +221,6 @@ -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED - -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED - -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED @@ -303,6 +302,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + true + + org.apache.maven.plugins @@ -366,13 +373,12 @@ spotless-maven-plugin ${spotless.version} - origin/master - - - + + 1.14.0 + @@ -385,6 +391,17 @@ maven-jar-plugin 3.2.0 + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + **/*Test.java + + false + + diff --git a/tensorflow-core/pom.xml b/tensorflow-core/pom.xml index 2d98332fa9f..1fd4c71debe 100644 --- a/tensorflow-core/pom.xml +++ b/tensorflow-core/pom.xml @@ -22,7 +22,7 @@ org.tensorflow tensorflow-java - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT tensorflow-core pom @@ -40,8 +40,10 @@ Match version used by TensorFlow, in https://github.com/tensorflow/tensorflow/blob/master/tensorflow/workspace2.bzl#L567 (but for the currently used release, not master) + + Bumped to newer version to patch a CVE only present in protobuf-java --> - 3.9.2 + 3.19.4 ${javacpp.platform}${javacpp.platform.extension} false @@ -65,7 +67,7 @@ macosx-x86_64${javacpp.platform.extension} windows-x86${javacpp.platform.extension} windows-x86_64${javacpp.platform.extension} - 1.5.6 + 1.5.7 diff --git a/tensorflow-core/tensorflow-core-api/.bazelrc b/tensorflow-core/tensorflow-core-api/.bazelrc index 26f501ff451..9f875293e54 100644 --- a/tensorflow-core/tensorflow-core-api/.bazelrc +++ b/tensorflow-core/tensorflow-core-api/.bazelrc @@ -1,2 +1,3 @@ build --remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm -build --remote_upload_local_results=false \ No newline at end of file +build --remote_upload_local_results=false +build --incompatible_restrict_string_escapes=false \ No newline at end of file diff --git a/tensorflow-core/tensorflow-core-api/.bazelversion b/tensorflow-core/tensorflow-core-api/.bazelversion index 0b2eb36f508..fae6e3d04b2 100644 --- a/tensorflow-core/tensorflow-core-api/.bazelversion +++ b/tensorflow-core/tensorflow-core-api/.bazelversion @@ -1 +1 @@ -3.7.2 +4.2.1 diff --git a/tensorflow-core/tensorflow-core-api/WORKSPACE b/tensorflow-core/tensorflow-core-api/WORKSPACE index f6aa07115ed..28f2bb27e6f 100644 --- a/tensorflow-core/tensorflow-core-api/WORKSPACE +++ b/tensorflow-core/tensorflow-core-api/WORKSPACE @@ -12,15 +12,16 @@ http_archive( # ":tensorflow-macosx.patch", # ":tensorflow-windows.patch", # https://github.com/tensorflow/tensorflow/issues/25213 ":tensorflow-proto.patch", + ":custom-grad-symbols.patch", ], patch_tool = "patch", patch_args = ["-p1"], patch_cmds = ["grep -rl 'java_package' tensorflow/core | xargs sed -i.bak 's/^\(.* java_package = \"org\.tensorflow\.\)\(.*\"\)/\\1proto.\\2'/"], urls = [ - "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.6.0.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.8.0.tar.gz", ], - sha256 = "41b32eeaddcbc02b0583660bcf508469550e4cd0f86b22d2abe72dfebeacde0f", - strip_prefix = "tensorflow-2.6.0" + sha256 = "66b953ae7fba61fd78969a2e24e350b26ec116cf2e6a7eb93d02c63939c6f9f7", + strip_prefix = "tensorflow-2.8.0" ) # START: Upstream TensorFlow dependencies diff --git a/tensorflow-core/tensorflow-core-api/build.sh b/tensorflow-core/tensorflow-core-api/build.sh index b5d25277ea0..368eb43d74f 100755 --- a/tensorflow-core/tensorflow-core-api/build.sh +++ b/tensorflow-core/tensorflow-core-api/build.sh @@ -7,9 +7,11 @@ export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 export BAZEL_VC="${VCINSTALLDIR:-}" if [[ -d $BAZEL_VC ]]; then + export BAZEL_BUILD="--output_user_root=$(cygpath -w $TMP) build" export BUILD_FLAGS="--copt=//arch:AVX `#--copt=//arch:AVX2` --define=override_eigen_strong_inline=true" export PYTHON_BIN_PATH=$(which python.exe) else + export BAZEL_BUILD="build" export BUILD_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx `#--copt=-mavx2 --copt=-mfma` --linkopt=-lstdc++ --host_linkopt=-lstdc++" export PYTHON_BIN_PATH=$(which python3) fi @@ -33,7 +35,7 @@ BUILD_FLAGS="$BUILD_FLAGS --experimental_repo_remote_exec --python_path="$PYTHON BUILD_FLAGS="$BUILD_FLAGS --distinct_host_configuration=true" # Build C/C++ API of TensorFlow itself including a target to generate ops for Java -bazel --bazelrc=tensorflow.bazelrc build $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \ +bazel --bazelrc=tensorflow.bazelrc $BAZEL_BUILD $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \ @org_tensorflow//tensorflow:tensorflow_cc \ @org_tensorflow//tensorflow/tools/lib_package:jnilicenses_generate \ :java_proto_gen_sources \ diff --git a/tensorflow-core/tensorflow-core-api/external/custom-grad-symbols.patch b/tensorflow-core/tensorflow-core-api/external/custom-grad-symbols.patch new file mode 100644 index 00000000000..c47b9da0127 --- /dev/null +++ b/tensorflow-core/tensorflow-core-api/external/custom-grad-symbols.patch @@ -0,0 +1,151 @@ +Index: tensorflow/tools/def_file_filter/BUILD +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD +--- a/tensorflow/tools/def_file_filter/BUILD (revision 5e5cc35b4c0f629a1e092b540fdf2b63367aa5ad) ++++ b/tensorflow/tools/def_file_filter/BUILD (date 1629063191558) +@@ -12,3 +12,8 @@ + name = "symbols_pybind", + srcs = ["symbols_pybind.txt"], + ) ++ ++filegroup( ++ name = "symbols_java", ++ srcs = ["symbols_java.txt"], ++) +Index: tensorflow/BUILD +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/tensorflow/BUILD b/tensorflow/BUILD +--- a/tensorflow/BUILD (revision 5e5cc35b4c0f629a1e092b540fdf2b63367aa5ad) ++++ b/tensorflow/BUILD (date 1629063361078) +@@ -1069,13 +1069,20 @@ + # the dynamic libraries of custom ops can find it at runtime. + genrule( + name = "tensorflow_filtered_def_file", +- srcs = [":tensorflow_def_file"], ++ srcs = [ ++ ":tensorflow_def_file", ++ ":java_symbol_target_libs_file", ++ ":win_lib_files_for_java_exported_symbols", ++ "//tensorflow/tools/def_file_filter:symbols_java", ++ ], + outs = ["tensorflow_filtered_def_file.def"], + cmd = select({ + "//tensorflow:windows": """ + $(location @local_config_def_file_filter//:def_file_filter) \\ + --input $(location :tensorflow_def_file) \\ +- --output $@ ++ --output $@ \\ ++ --symbols $(location //tensorflow/tools/def_file_filter:symbols_java) \\ ++ --lib_paths_file $(location :java_symbol_target_libs_file) + """, + "//conditions:default": "touch $@", # Just a placeholder for Unix platforms + }), +@@ -1083,6 +1090,34 @@ + visibility = ["//visibility:public"], + ) + ++# Write to a file a list of all cc_library targets that we need for exporting symbols on Windows. ++genrule( ++ name = "java_symbol_target_libs_file", ++ srcs = [":win_lib_files_for_java_exported_symbols"], ++ outs = ["java_symbol_target_libs_file.txt"], ++ cmd = select({ ++ "//tensorflow:windows": """ ++ for SRC in $(SRCS); do ++ echo $$SRC | sed 's/third_party\\///g' >> $@ ++ done ++ """, ++ "//conditions:default": "touch $@", # Just a placeholder for Unix platforms ++ }), ++ visibility = ["//visibility:public"], ++) ++ ++filegroup( ++ name = "win_lib_files_for_java_exported_symbols", ++ srcs = [ ++ "//tensorflow/cc:scope", ++ "//tensorflow/cc:grad_op_registry", ++ "//tensorflow/c:tf_status_helper", ++ "//tensorflow/cc:ops" ++ ], ++ visibility = ["//visibility:private"], ++) ++ ++ + # The interface library (tensorflow.dll.if.lib) for linking tensorflow DLL library (tensorflow.dll) on Windows. + # To learn more about import library (called interface library in Bazel): + # https://docs.microsoft.com/en-us/cpp/build/linking-an-executable-to-a-dll?view=vs-2017#linking-implicitly +Index: tensorflow/tools/def_file_filter/BUILD.tpl +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl +--- a/tensorflow/tools/def_file_filter/BUILD.tpl (revision 5e5cc35b4c0f629a1e092b540fdf2b63367aa5ad) ++++ b/tensorflow/tools/def_file_filter/BUILD.tpl (date 1629063191583) +@@ -18,3 +18,8 @@ + name = "symbols_pybind", + srcs = ["symbols_pybind.txt"], + ) ++ ++filegroup( ++ name = "symbols_java", ++ srcs = ["symbols_java.txt"], ++) +Index: tensorflow/tools/def_file_filter/symbols_java.txt +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/tensorflow/tools/def_file_filter/symbols_java.txt b/tensorflow/tools/def_file_filter/symbols_java.txt +new file mode 100644 +--- /dev/null (date 1629063607794) ++++ b/tensorflow/tools/def_file_filter/symbols_java.txt (date 1629063607794) +@@ -0,0 +1,26 @@ ++[//tensorflow/cc:scope] # scope ++tensorflow::Scope::graph ++tensorflow::Scope::ok ++tensorflow::Scope::UpdateBuilder ++tensorflow::Scope::GetUniqueNameForOp ++tensorflow::Scope::ExitOnError ++tensorflow::Scope::WithDevice ++tensorflow::Scope::WithNoControlDependencies ++tensorflow::Scope::WithControlDependencies ++tensorflow::Scope::NewSubScope ++tensorflow::Scope::NewRootScope ++tensorflow::Scope::operator= ++tensorflow::Scope::~Scope ++tensorflow::Scope::Scope ++ ++[//tensorflow/cc:ops] ++tensorflow::Operation::Operation ++ ++[//tensorflow/cc:grad_op_registry] # custom gradients for graph ++tensorflow::ops::GradOpRegistry::Global ++tensorflow::ops::GradOpRegistry::Lookup ++tensorflow::ops::GradOpRegistry::Register ++ ++[//tensorflow/c:tf_status_helper] # status helpers ++tensorflow::Set_TF_Status_from_Status ++tensorflow::StatusFromTF_Status +=================================================================== +diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl +--- a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl (revision 919f693420e35d00c8d0a42100837ae3718f7927) ++++ b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl (date 1632048268359) +@@ -143,8 +143,8 @@ + re_filter_comp = re.compile(r"{}".format(re_filter)) + + # Filter out symbol from the split line (`sym_split` in the for loop below). +- sym_line_filter = r".*\s+\| (.*) \(.*" +- sym_line_filter_anomaly = r".*\s+\| (.*)" ++ sym_line_filter = r".*\s+\| (.*?) \(.*" ++ sym_line_filter_anomaly = r".*\s+\| (.*?)" + + for sym_line in sym_split: + if re_filter_comp.search(sym_line): diff --git a/tensorflow-core/tensorflow-core-api/external/tensorflow-proto.patch b/tensorflow-core/tensorflow-core-api/external/tensorflow-proto.patch index 7823514e4bc..97a01022aa0 100644 --- a/tensorflow-core/tensorflow-core-api/external/tensorflow-proto.patch +++ b/tensorflow-core/tensorflow-core-api/external/tensorflow-proto.patch @@ -1,6 +1,92 @@ -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/bfc_memory_map.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/bfc_memory_map.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/bfc_memory_map.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/bfc_memory_map.proto 2021-08-30 11:22:48.263351451 +0900 +diff -ruN tensorflow-2.7.0/tensorflow/core/framework/dataset_metadata.proto tensorflow-2.7.0-proto/tensorflow/core/framework/dataset_metadata.proto +--- tensorflow-2.7.0/tensorflow/core/framework/dataset_metadata.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/framework/dataset_metadata.proto 2021-11-09 12:11:55.183453737 +0900 +@@ -2,6 +2,7 @@ + + package tensorflow.data; + ++option java_package = "org.tensorflow.data"; + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_metadata_go_proto"; + + // next: 2 +diff -ruN tensorflow-2.7.0/tensorflow/core/framework/dataset_options.proto tensorflow-2.7.0-proto/tensorflow/core/framework/dataset_options.proto +--- tensorflow-2.7.0/tensorflow/core/framework/dataset_options.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/framework/dataset_options.proto 2021-11-09 12:07:40.449571619 +0900 +@@ -4,6 +4,10 @@ + + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_options_go_proto"; + ++option java_outer_classname = "DatasetOptionsProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.data"; ++ + // Represents the type of auto-sharding we enable. + enum AutoShardPolicy { + // AUTO: Attempts FILE-based sharding, falling back to DATA-based sharding. +diff -ruN tensorflow-2.7.0/tensorflow/core/framework/model.proto tensorflow-2.7.0-proto/tensorflow/core/framework/model.proto +--- tensorflow-2.7.0/tensorflow/core/framework/model.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/framework/model.proto 2021-11-09 12:07:40.450571622 +0900 +@@ -3,6 +3,9 @@ + package tensorflow.data.model; + + option cc_enable_arenas = true; ++option java_outer_classname = "ModelProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.data.model"; + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/model_go_proto"; + + // Class of a node in the performance model. +diff -ruN tensorflow-2.7.0/tensorflow/core/grappler/costs/op_performance_data.proto tensorflow-2.7.0-proto/tensorflow/core/grappler/costs/op_performance_data.proto +--- tensorflow-2.7.0/tensorflow/core/grappler/costs/op_performance_data.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/grappler/costs/op_performance_data.proto 2021-11-09 12:07:40.450571622 +0900 +@@ -17,6 +17,9 @@ + + package tensorflow; + option cc_enable_arenas = true; ++option java_outer_classname = "OpPerformanceDataProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.framework"; + + import "tensorflow/core/framework/tensor.proto"; + import "tensorflow/core/framework/tensor_shape.proto"; +diff -ruN tensorflow-2.7.0/tensorflow/core/lib/core/error_codes.proto tensorflow-2.7.0-proto/tensorflow/core/lib/core/error_codes.proto +--- tensorflow-2.7.0/tensorflow/core/lib/core/error_codes.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/lib/core/error_codes.proto 2021-11-09 12:07:40.447571613 +0900 +@@ -1,3 +1,5 @@ + syntax = "proto3"; + ++option java_package = "org.tensorflow.framework"; ++ + import public "tensorflow/core/protobuf/error_codes.proto"; +diff -ruN tensorflow-2.7.0/tensorflow/core/profiler/profiler_options.proto tensorflow-2.7.0-proto/tensorflow/core/profiler/profiler_options.proto +--- tensorflow-2.7.0/tensorflow/core/profiler/profiler_options.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/profiler/profiler_options.proto 2021-11-09 12:07:40.448571616 +0900 +@@ -1,6 +1,9 @@ + syntax = "proto3"; + + package tensorflow; ++option java_outer_classname = "ProfilerOptionsProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.profiler"; + + // Next ID: 11 + message ProfileOptions { +diff -ruN tensorflow-2.7.0/tensorflow/core/profiler/protobuf/xplane.proto tensorflow-2.7.0-proto/tensorflow/core/profiler/protobuf/xplane.proto +--- tensorflow-2.7.0/tensorflow/core/profiler/protobuf/xplane.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/profiler/protobuf/xplane.proto 2021-11-09 12:07:40.447571613 +0900 +@@ -3,6 +3,9 @@ + package tensorflow.profiler; + + option cc_enable_arenas = true; ++option java_outer_classname = "XPlaneProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.profiler"; + + // A container of parallel XPlanes, generated by one or more profiling sources. + // Next ID: 5 +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/bfc_memory_map.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/bfc_memory_map.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/bfc_memory_map.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/bfc_memory_map.proto 2021-11-09 12:07:40.443571601 +0900 @@ -3,6 +3,9 @@ package tensorflow; @@ -11,23 +97,32 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/bfc_memory_map.proto tensorf // Some of the data from AllocatorStats message MemAllocatorStats { -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/snapshot.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/snapshot.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/snapshot.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/snapshot.proto 2021-08-30 11:22:48.264351453 +0900 -@@ -8,6 +8,10 @@ +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/composite_tensor_variant.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/composite_tensor_variant.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/composite_tensor_variant.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/composite_tensor_variant.proto 2021-11-09 12:07:40.451571625 +0900 +@@ -3,7 +3,7 @@ + package tensorflow; + import "tensorflow/core/protobuf/struct.proto"; +- ++option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; -+option java_outer_classname = "SnapshotProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.data.experimental"; -+ - // Each SnapshotRecord represents one batch of pre-processed input data. A batch - // consists of a list of tensors that we encode as TensorProtos. This message - // doesn't store the structure of the batch. -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/device_properties.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/device_properties.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/device_properties.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/device_properties.proto 2021-08-30 11:22:48.264351453 +0900 + // Metadata for CompositeTensorVariant, used when serializing as Variant. +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/data_service.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/data_service.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/data_service.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/data_service.proto 2021-11-09 12:10:45.915184828 +0900 +@@ -2,6 +2,7 @@ + + package tensorflow.data; + ++option java_package = "org.tensorflow.data"; + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + + message ProcessingModeDef { +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/device_properties.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/device_properties.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/device_properties.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/device_properties.proto 2021-11-09 12:07:40.444571604 +0900 @@ -19,6 +19,8 @@ option cc_enable_arenas = true; @@ -37,10 +132,10 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/device_properties.proto tens option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; message DeviceProperties { -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/saved_object_graph.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/saved_object_graph.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/saved_object_graph.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/saved_object_graph.proto 2021-08-30 11:22:48.265351456 +0900 -@@ -11,6 +11,9 @@ +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/saved_object_graph.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/saved_object_graph.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/saved_object_graph.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/saved_object_graph.proto 2021-11-09 12:07:40.445571607 +0900 +@@ -12,6 +12,9 @@ option cc_enable_arenas = true; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; @@ -50,9 +145,275 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/saved_object_graph.proto ten // A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It // describes the directed graph of Python objects (or equivalent in other -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/struct.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/struct.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/struct.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/struct.proto 2021-08-30 11:22:48.265351456 +0900 +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/saved_object_graph.proto.orig tensorflow-2.7.0-proto/tensorflow/core/protobuf/saved_object_graph.proto.orig +--- tensorflow-2.7.0/tensorflow/core/protobuf/saved_object_graph.proto.orig 1970-01-01 09:00:00.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/saved_object_graph.proto.orig 2021-11-01 10:31:04.000000000 +0900 +@@ -0,0 +1,225 @@ ++syntax = "proto3"; ++ ++package tensorflow; ++ ++import "google/protobuf/any.proto"; ++import "tensorflow/core/framework/tensor_shape.proto"; ++import "tensorflow/core/framework/types.proto"; ++import "tensorflow/core/framework/variable.proto"; ++import "tensorflow/core/framework/versions.proto"; ++import "tensorflow/core/protobuf/struct.proto"; ++import "tensorflow/core/protobuf/trackable_object_graph.proto"; ++ ++option cc_enable_arenas = true; ++option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; ++ ++// A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It ++// describes the directed graph of Python objects (or equivalent in other ++// languages) that make up a model, with nodes[0] at the root. ++ ++// SavedObjectGraph shares some structure with TrackableObjectGraph, but ++// SavedObjectGraph belongs to the MetaGraph and contains pointers to functions ++// and type information, while TrackableObjectGraph lives in the checkpoint ++// and contains pointers only to variable values. ++ ++message SavedObjectGraph { ++ // Flattened list of objects in the object graph. ++ // ++ // The position of the object in this list indicates its id. ++ // Nodes[0] is considered the root node. ++ repeated SavedObject nodes = 1; ++ ++ // Information about captures and output structures in concrete functions. ++ // Referenced from SavedBareConcreteFunction and SavedFunction. ++ map concrete_functions = 2; ++} ++ ++message SavedObject { ++ // Objects which this object depends on: named edges in the dependency ++ // graph. ++ // ++ // Note: currently only valid if kind == "user_object" or "resource". ++ repeated TrackableObjectGraph.TrackableObject.ObjectReference children = 1; ++ ++ // Removed when forking SavedObject from TrackableObjectGraph. ++ reserved "attributes"; ++ reserved 2; ++ ++ // Slot variables owned by this object. This describes the three-way ++ // (optimizer, variable, slot variable) relationship; none of the three ++ // depend on the others directly. ++ // ++ // Note: currently only valid if kind == "user_object". ++ repeated TrackableObjectGraph.TrackableObject.SlotVariableReference ++ slot_variables = 3; ++ ++ oneof kind { ++ SavedUserObject user_object = 4; ++ SavedAsset asset = 5; ++ SavedFunction function = 6; ++ SavedVariable variable = 7; ++ SavedBareConcreteFunction bare_concrete_function = 8; ++ SavedConstant constant = 9; ++ SavedResource resource = 10; ++ CapturedTensor captured_tensor = 12; ++ } ++ ++ map saveable_objects = 11; ++ ++ // The fields below are filled when the user serializes a registered Trackable ++ // class. Registered classes may save additional metadata and supersede the ++ // default loading process where nodes are recreated from the proto. ++ // ++ // The name of the registered class of the form "{package}.{class_name}". ++ // This field is used to search for the registered class at loading time. ++ string registered_name = 13; ++ // The user-generated proto storing metadata for this object, to be passed to ++ // the registered classes's _deserialize_from_proto method when this object is ++ // loaded from the SavedModel. ++ google.protobuf.Any serialized_user_proto = 14; ++} ++ ++// A SavedUserObject is an object (in the object-oriented language of the ++// TensorFlow program) of some user- or framework-defined class other than ++// those handled specifically by the other kinds of SavedObjects. ++// ++// This object cannot be evaluated as a tensor, and therefore cannot be bound ++// to an input of a function. ++message SavedUserObject { ++ // Corresponds to a registration of the type to use in the loading program. ++ string identifier = 1; ++ // Version information from the producer of this SavedUserObject. ++ VersionDef version = 2; ++ // Metadata for deserializing this object. ++ // ++ // Deprecated! At the time of deprecation, Keras was the only user of this ++ // field, and its saving and loading code will be updated shortly. ++ // Please save your application-specific metadata to a separate file. ++ string metadata = 3 [deprecated = true]; ++} ++ ++// A SavedAsset points to an asset in the MetaGraph. ++// ++// When bound to a function this object evaluates to a tensor with the absolute ++// filename. Users should not depend on a particular part of the filename to ++// remain stable (e.g. basename could be changed). ++message SavedAsset { ++ // Index into `MetaGraphDef.asset_file_def[]` that describes the Asset. ++ // ++ // Only the field `AssetFileDef.filename` is used. Other fields, such as ++ // `AssetFileDef.tensor_info`, MUST be ignored. ++ int32 asset_file_def_index = 1; ++} ++ ++// A function with multiple signatures, possibly with non-Tensor arguments. ++message SavedFunction { ++ repeated string concrete_functions = 1; ++ FunctionSpec function_spec = 2; ++} ++ ++message CapturedTensor { ++ // Name of captured tensor ++ string name = 1; ++ ++ // Name of concrete function which contains the computed graph tensor. ++ string concrete_function = 2; ++} ++ ++// Stores low-level information about a concrete function. Referenced in either ++// a SavedFunction or a SavedBareConcreteFunction. ++message SavedConcreteFunction { ++ repeated int32 bound_inputs = 2; ++ ++ // Input in canonicalized form that was received to create this concrete ++ // function. ++ StructuredValue canonicalized_input_signature = 3; ++ // Output that was the return value of this function after replacing all ++ // Tensors with TensorSpecs. This can be an arbitrary nested function and will ++ // be used to reconstruct the full structure from pure tensors. ++ StructuredValue output_signature = 4; ++} ++ ++message SavedBareConcreteFunction { ++ // Identifies a SavedConcreteFunction. ++ string concrete_function_name = 1; ++ ++ // A sequence of unique strings, one per Tensor argument. ++ repeated string argument_keywords = 2; ++ // The prefix of `argument_keywords` which may be identified by position. ++ int64 allowed_positional_arguments = 3; ++ // The spec of the function that this ConcreteFunction is traced from. This ++ // allows the ConcreteFunction to be called with nest structure inputs. This ++ // field may not be populated. If this field is absent, the concrete function ++ // can only be called with flat inputs. ++ // TODO(b/169361281): support calling saved ConcreteFunction with structured ++ // inputs in C++ SavedModel API. ++ FunctionSpec function_spec = 4; ++} ++ ++message SavedConstant { ++ // An Operation name for a ConstantOp in this SavedObjectGraph's MetaGraph. ++ string operation = 1; ++} ++ ++// Represents a Variable that is initialized by loading the contents from the ++// checkpoint. ++message SavedVariable { ++ DataType dtype = 1; ++ TensorShapeProto shape = 2; ++ bool trainable = 3; ++ VariableSynchronization synchronization = 4; ++ VariableAggregation aggregation = 5; ++ string name = 6; ++ string device = 7; ++ // List of component variables for a distributed variable. ++ // ++ // When this field is non-empty, the SavedVariable will be assumed ++ // to be a distributed variable defined by the components listed here. ++ // ++ // This is only supported by experimental loaders at the moment. ++ repeated SavedVariable experimental_distributed_variable_components = 8; ++} ++ ++// Represents `FunctionSpec` used in `Function`. This represents a ++// function that has been wrapped as a TensorFlow `Function`. ++message FunctionSpec { ++ // Full arg spec from inspect.getfullargspec(). ++ StructuredValue fullargspec = 1; ++ // Whether this represents a class method. ++ bool is_method = 2; ++ // The input signature, if specified. ++ StructuredValue input_signature = 5; ++ ++ // Whether the function should be compiled by XLA. ++ // ++ // The public interface to `tf.function` uses an optional boolean to ++ // represent three distinct states for this field. Unfortunately, proto3 ++ // removes the ability to explicitly check for the presence or absence of a ++ // field, so we instead map to an enum. ++ // ++ // See `tf.function` for details. ++ enum JitCompile { ++ DEFAULT = 0; ++ ON = 1; ++ OFF = 2; ++ } ++ JitCompile jit_compile = 6; ++ ++ reserved 3, 4; ++} ++ ++// A SavedResource represents a TF object that holds state during its lifetime. ++// An object of this type can have a reference to a: ++// create_resource() and an initialize() function. ++message SavedResource { ++ // A device specification indicating a required placement for the resource ++ // creation function, e.g. "CPU". An empty string allows the user to select a ++ // device. ++ string device = 1; ++} ++ ++message SaveableObject { ++ // Node ids of concrete functions for saving and loading from a checkpoint. ++ int32 save_function = 2; ++ int32 restore_function = 3; ++} +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/service_config.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/service_config.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/service_config.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/service_config.proto 2021-11-09 12:07:40.449571619 +0900 +@@ -1,6 +1,7 @@ + syntax = "proto3"; + + package tensorflow.data.experimental; ++option java_package = "org.tensorflow.data.experimental"; + + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/snapshot.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/snapshot.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/snapshot.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/snapshot.proto 2021-11-09 12:07:40.444571604 +0900 +@@ -8,6 +8,10 @@ + + option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + ++option java_outer_classname = "SnapshotProtos"; ++option java_multiple_files = true; ++option java_package = "org.tensorflow.data.experimental"; ++ + // Each SnapshotRecord represents one batch of pre-processed input data. A batch + // consists of a list of tensors that we encode as TensorProtos. This message + // doesn't store the structure of the batch. +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/status.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/status.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/status.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/status.proto 2021-11-09 12:09:16.716853813 +0900 +@@ -2,6 +2,8 @@ + + package tensorflow; + ++option java_package = "org.tensorflow.framework"; ++ + // If included as a payload, this message flags the Status to be a "derived" + // Status. Used by StatusGroup to ignore certain Statuses when reporting + // errors to end users. +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/struct.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/struct.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/struct.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/struct.proto 2021-11-09 12:07:40.445571607 +0900 @@ -7,6 +7,9 @@ import "tensorflow/core/framework/types.proto"; @@ -63,9 +424,9 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/struct.proto tensorflow-2.6. // `StructuredValue` represents a dynamically typed value representing various // data structures that are inspired by Python data structures typically used in -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/trackable_object_graph.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/trackable_object_graph.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/trackable_object_graph.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/trackable_object_graph.proto 2021-08-30 11:22:48.266351458 +0900 +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/trackable_object_graph.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/trackable_object_graph.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/trackable_object_graph.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/trackable_object_graph.proto 2021-11-09 12:07:40.446571610 +0900 @@ -4,6 +4,9 @@ option cc_enable_arenas = true; @@ -76,9 +437,9 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/trackable_object_graph.proto // A TensorBundle addition which saves extra information about the objects which // own variables, allowing for more robust checkpoint loading into modified -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/transport_options.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/transport_options.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/transport_options.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/transport_options.proto 2021-08-30 11:22:48.266351458 +0900 +diff -ruN tensorflow-2.7.0/tensorflow/core/protobuf/transport_options.proto tensorflow-2.7.0-proto/tensorflow/core/protobuf/transport_options.proto +--- tensorflow-2.7.0/tensorflow/core/protobuf/transport_options.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/protobuf/transport_options.proto 2021-11-09 12:07:40.446571610 +0900 @@ -3,6 +3,7 @@ package tensorflow; @@ -87,31 +448,9 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/transport_options.proto tens // Extra data needed on a non-RDMA RecvBufResponse. message RecvBufRespExtra { -diff -ruN tensorflow-2.6.0/tensorflow/core/lib/core/error_codes.proto tensorflow-2.6.0-proto/tensorflow/core/lib/core/error_codes.proto ---- tensorflow-2.6.0/tensorflow/core/lib/core/error_codes.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/lib/core/error_codes.proto 2021-08-30 11:22:48.267351461 +0900 -@@ -1,3 +1,5 @@ - syntax = "proto3"; - -+option java_package = "org.tensorflow.framework"; -+ - import public "tensorflow/core/protobuf/error_codes.proto"; -diff -ruN tensorflow-2.6.0/tensorflow/core/profiler/protobuf/xplane.proto tensorflow-2.6.0-proto/tensorflow/core/profiler/protobuf/xplane.proto ---- tensorflow-2.6.0/tensorflow/core/profiler/protobuf/xplane.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/profiler/protobuf/xplane.proto 2021-08-30 11:22:48.267351461 +0900 -@@ -3,6 +3,9 @@ - package tensorflow.profiler; - - option cc_enable_arenas = true; -+option java_outer_classname = "XPlaneProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.profiler"; - - // A container of parallel XPlanes, generated by one or more profiling sources. - // Next ID: 5 -diff -ruN tensorflow-2.6.0/tensorflow/core/util/memmapped_file_system.proto tensorflow-2.6.0-proto/tensorflow/core/util/memmapped_file_system.proto ---- tensorflow-2.6.0/tensorflow/core/util/memmapped_file_system.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/util/memmapped_file_system.proto 2021-08-30 11:22:48.268351463 +0900 +diff -ruN tensorflow-2.7.0/tensorflow/core/util/memmapped_file_system.proto tensorflow-2.7.0-proto/tensorflow/core/util/memmapped_file_system.proto +--- tensorflow-2.7.0/tensorflow/core/util/memmapped_file_system.proto 2021-11-01 10:31:04.000000000 +0900 ++++ tensorflow-2.7.0-proto/tensorflow/core/util/memmapped_file_system.proto 2021-11-09 12:07:40.448571616 +0900 @@ -17,6 +17,9 @@ package tensorflow; @@ -122,79 +461,25 @@ diff -ruN tensorflow-2.6.0/tensorflow/core/util/memmapped_file_system.proto tens // A message that describes one region of memmapped file. message MemmappedFileSystemDirectoryElement { -diff -ruN tensorflow-2.6.0/tensorflow/core/profiler/profiler_options.proto tensorflow-2.6.0-proto/tensorflow/core/profiler/profiler_options.proto ---- tensorflow-2.6.0/tensorflow/core/profiler/profiler_options.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/profiler/profiler_options.proto 2021-08-30 11:22:48.268351463 +0900 -@@ -1,6 +1,9 @@ - syntax = "proto3"; - +diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto +--- tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto 2022-02-01 04:17:33.000000000 +0900 ++++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto 2022-02-16 11:02:10.162709816 +0900 +@@ -3,6 +3,7 @@ package tensorflow; -+option java_outer_classname = "ProfilerOptionsProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.profiler"; - - // Next ID: 11 - message ProfileOptions { -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/service_config.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/service_config.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/service_config.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/service_config.proto 2021-08-30 11:22:48.269351466 +0900 -@@ -1,6 +1,7 @@ - syntax = "proto3"; - - package tensorflow.data.experimental; -+option java_package = "org.tensorflow.data.experimental"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; ++option java_package = "org.tensorflow.distruntime"; -diff -ruN tensorflow-2.6.0/tensorflow/core/framework/dataset_options.proto tensorflow-2.6.0-proto/tensorflow/core/framework/dataset_options.proto ---- tensorflow-2.6.0/tensorflow/core/framework/dataset_options.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/framework/dataset_options.proto 2021-08-30 11:22:48.269351466 +0900 -@@ -4,6 +4,10 @@ - - option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_options_go_proto"; - -+option java_outer_classname = "DatasetOptionsProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.data"; -+ - // Represents the type of auto-sharding we enable. - enum AutoShardPolicy { - // AUTO: Attempts FILE-based sharding, falling back to DATA-based sharding. -diff -ruN tensorflow-2.6.0/tensorflow/core/framework/model.proto tensorflow-2.6.0-proto/tensorflow/core/framework/model.proto ---- tensorflow-2.6.0/tensorflow/core/framework/model.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/framework/model.proto 2021-08-30 11:23:28.579451037 +0900 -@@ -3,6 +3,9 @@ - package tensorflow.data.model; - - option cc_enable_arenas = true; -+option java_outer_classname = "ModelProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.data.model"; - option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/model_go_proto"; - - // Class of a node in the performance model. -diff -ruN tensorflow-2.6.0/tensorflow/core/grappler/costs/op_performance_data.proto tensorflow-2.6.0-proto/tensorflow/core/grappler/costs/op_performance_data.proto ---- tensorflow-2.6.0/tensorflow/core/grappler/costs/op_performance_data.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/grappler/costs/op_performance_data.proto 2021-08-30 11:22:48.270351468 +0900 -@@ -17,6 +17,9 @@ + // Coordination service configuration parameters. + // The system picks appropriate values for fields that are not set. +diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto +--- tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-01 04:17:33.000000000 +0900 ++++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-16 11:00:03.739373379 +0900 +@@ -4,6 +4,7 @@ - package tensorflow; option cc_enable_arenas = true; -+option java_outer_classname = "OpPerformanceDataProtos"; -+option java_multiple_files = true; -+option java_package = "org.tensorflow.framework"; - - import "tensorflow/core/framework/tensor.proto"; - import "tensorflow/core/framework/tensor_shape.proto"; -diff -ruN tensorflow-2.6.0/tensorflow/core/protobuf/composite_tensor_variant.proto tensorflow-2.6.0-proto/tensorflow/core/protobuf/composite_tensor_variant.proto ---- tensorflow-2.6.0/tensorflow/core/protobuf/composite_tensor_variant.proto 2021-08-10 04:10:27.000000000 +0900 -+++ tensorflow-2.6.0-proto/tensorflow/core/protobuf/composite_tensor_variant.proto 2021-08-30 15:43:37.086090343 +0900 -@@ -3,7 +3,7 @@ - package tensorflow; - - import "tensorflow/core/protobuf/struct.proto"; -- -+option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; ++option java_package = "org.tensorflow.distruntime"; - // Metadata for CompositeTensorVariant, used when serializing as Variant. + // Used to serialize and transmit tensorflow::Status payloads through + // grpc::Status `error_details` since grpc::Status lacks payload API. diff --git a/tensorflow-core/tensorflow-core-api/pom.xml b/tensorflow-core/tensorflow-core-api/pom.xml index 9f23757e83d..6f793b49dd5 100644 --- a/tensorflow-core/tensorflow-core-api/pom.xml +++ b/tensorflow-core/tensorflow-core-api/pom.xml @@ -6,7 +6,7 @@ org.tensorflow tensorflow-core - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT tensorflow-core-api jar @@ -20,7 +20,7 @@ ${native.build.skip} ${native.build.skip} org.tensorflow.core.api - 0.3.3 + 0.4.0-SNAPSHOT 1.0.1 @@ -78,8 +78,8 @@ dev - - - org.tensorflow - tensorflow-core-api - ${project.version} - ${native.classifier} - - true + + + + org.apache.maven.plugins + maven-dependency-plugin + + + dev-unpack-native + initialize + + unpack + + + ${project.groupId}:${project.artifactId}:${project.version}:jar:${native.classifier} + ${project.build.directory}/native + + + + + + + + javacpp-parser + initialize + + resources + + + + maven-compiler-plugin 3.8.0 @@ -171,12 +201,14 @@ javacpp-parser - generate-sources + initialize compile @@ -184,6 +216,9 @@ org/tensorflow/internal/c_api/presets/*.java + + 8 + @@ -212,6 +247,11 @@ ${project.basedir}/ ${project.basedir}/bazel-${project.artifactId}/external/org_tensorflow/ + ${project.basedir}/bazel-bin/external/org_tensorflow/ + ${project.basedir}/bazel-${project.artifactId}/external/com_google_absl/ + ${project.basedir}/bazel-${project.artifactId}/external/eigen_archive/ + ${project.basedir}/bazel-${project.artifactId}/external/com_google_protobuf/src/ + ${project.basedir}/target/classes/org/tensorflow/internal/c_api/include/ ${project.basedir}/bazel-bin/external/llvm_openmp/ @@ -305,7 +345,7 @@ javacpp-compiler @@ -409,7 +449,6 @@ maven-surefire-plugin - 3.0.0-M5