From 1222a58b6ba15396ca7649e79cad833a9b2f6071 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Thu, 13 Feb 2025 10:18:13 -0800 Subject: [PATCH 01/14] jar build workflow --- .github/workflows/jarbuild.yml | 615 +++++++++++++++++++++++++++++++++ 1 file changed, 615 insertions(+) create mode 100644 .github/workflows/jarbuild.yml diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml new file mode 100644 index 0000000000..5ba080fd8a --- /dev/null +++ b/.github/workflows/jarbuild.yml @@ -0,0 +1,615 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + workflow_dispatch: + inputs: + arrow_branch: + description: 'Arrow branch to build' + required: true + arrow_repo: + description: 'Arrow repository' + type: string + required: true + default: https://github.com/dremio/arrow.git + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + source: + name: Source + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Prepare for tag + if: github.ref_type == 'tag' + run: | + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Prepare for branch + if: github.ref_type == 'branch' + run: | + version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + rc=$(date +%Y%m%d) + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Archive + run: | + id="apache-arrow-java-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - name: Upload source archive + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-source + path: | + apache-arrow-java-* + jni-linux: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 120 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: ubuntu-latest + arch: "x86_64" + archery_arch: "amd64" + - runs_on: ubuntu-24.04-arm + arch: "aarch_64" + archery_arch: "arm64v8" + env: + # architecture name used for archery build + ARCH: ${{ matrix.platform.archery_arch }} + DOCKER_VOLUME_PREFIX: .docker/ + permissions: + contents: read + packages: write + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .docker + key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-linux-${{ matrix.platform.arch }}- + - name: Build + run: | + docker compose run vcpkg-jni + - name: Push Docker image + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main' + run: | + docker compose push vcpkg-jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-linux-${{ matrix.platform.arch }} + path: jni-linux-${{ matrix.platform.arch }}.tar.gz + jni-macos: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-14, arch: "aarch_64" } + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + cache: 'pip' + python-version: 3.12 + - name: Install Archery + run: pip install -e arrow/dev/archery[all] + - name: Install dependencies + run: | + # We want to use llvm@14 to avoid shared z3 + # dependency. llvm@14 doesn't depend on z3 and llvm depends + # on z3. And Homebrew's z3 provides only shared library. It + # doesn't provides static z3 because z3's CMake doesn't accept + # building both shared and static libraries at once. + # See also: Z3_BUILD_LIBZ3_SHARED in + # https://github.com/Z3Prover/z3/blob/master/README-CMake.md + # + # If llvm is installed, Apache Arrow C++ uses llvm rather than + # llvm@14 because llvm is newer than llvm@14. + brew uninstall llvm || : + + # Ensure updating python@XXX with the "--overwrite" option. + # If python@XXX is updated without "--overwrite", it causes + # a conflict error. Because Python 3 installed not by + # Homebrew exists in /usr/local on GitHub Actions. If + # Homebrew's python@XXX is updated without "--overwrite", it + # tries to replace /usr/local/bin/2to3 and so on and causes + # a conflict error. + brew update + for python_package in $(brew list | grep python@); do + brew install --overwrite ${python_package} + done + brew install --overwrite python + + if [ "$(uname -m)" = "arm64" ]; then + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : + fi + + brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + + brew bundle --file=Brewfile + - name: Prepare ccache + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-macos-${{ matrix.platform.arch }}- + - name: Build + run: | + set -e + # make brew Java available to CMake + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + ci/scripts/jni_macos_build.sh . arrow build jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-macos-${{ matrix.platform.arch }} + path: jni-macos-${{ matrix.platform.arch }}.tar.gz + jni-windows: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: windows-2019 + arch: "x86_64" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + shell: bash + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # shell: bash + # run: | + # ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + # if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow + - name: Set up Java + uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 + with: + java-version: '11' + distribution: 'temurin' + - name: Download Timezone Database + shell: bash + run: | + arrow/ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: | + env | sort + version=4.10.2 + base_name="ccache-${version}-windows-x86_64" + url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip" + curl --fail --location --remote-name "${url}" + unzip "${base_name}.zip" + chmod +x "${base_name}/ccache.exe" + mv "${base_name}/ccache.exe" /usr/bin/ + rm -rf "${base_name}"{,.zip} + - name: Prepare ccache + shell: bash + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-windows-${{ matrix.platform.arch }}- + - name: Build + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM For ORC + set TZDIR=/c/msys64/usr/share/zoneinfo + bash -c "ci/scripts/jni_windows_build.sh . arrow build jni" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-windows-${{ matrix.platform.arch }} + path: jni-windows-${{ matrix.platform.arch }}.tar.gz + binaries: + name: Binaries + runs-on: ubuntu-latest + needs: + - jni-linux + - jni-macos + - jni-windows + steps: + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: artifacts + - name: Decompress artifacts + run: | + mv artifacts/*/*.tar.gz . + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + tar -xvzf jni-linux-x86_64.tar.gz + tar -xvzf jni-linux-aarch_64.tar.gz + tar -xvzf jni-macos-x86_64.tar.gz + tar -xvzf jni-macos-aarch_64.tar.gz + tar -xvzf jni-windows-x86_64.tar.gz + - name: Test that shared libraries exist + run: | + set -x + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/x86_64/libgandiva_jni.so + + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib + + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib + + test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: testing + - name: Cache ~/.m2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ~/.m2 + key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} + restore-keys: binaries-build- + - name: Build bundled JAR and docs + run: | + ci/scripts/jni_full_build.sh . jni binaries + - name: Prepare docs + run: | + mkdir -p docs + cp -a target/site/apidocs reference + tar -cvzf reference.tar.gz reference + - name: Upload binaries + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-binaries + path: binaries/* + - name: Upload docs + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: reference + path: reference.tar.gz + docs: + name: Docs + needs: + - binaries + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + cache: 'pip' + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Download Javadocs + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: reference + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Build + run: | + cd docs + python -m venv venv + source venv/bin/activate + pip install -r requirements.txt + make html + tar -xf ../reference.tar.gz -C build/html + - name: Compress into single artifact to keep directory structure + run: tar -cvzf html.tar.gz -C docs/build html + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-html + path: html.tar.gz + publish-docs: + name: Publish docs + # Run only when: + # * We push to a branch + # * If the target repository is apache/arrow-java: + # * The target branch is main + # * Else (fork repositories): + # * All branches + # * We can preview the last pushed content + # at https://${YOUR_GITHUB_ACCOUNT}.github.io/arrow-java/main/ + if: >- + github.event_name == 'push' && + github.ref_type == 'branch' && + ((github.repository == 'apache/arrow-java' && github.ref_name == 'main') || + github.repository != 'apache/arrow-java') + needs: + - docs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: site + - name: Prepare branch + run: | + if [ "${GITHUB_REPOSITORY}" = "apache/arrow-java" ]; then + BRANCH=asf-site + else + BRANCH=gh-pages + fi + echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV} + + cp site/.asf.yaml ./ + cd site + git fetch + if ! git switch -c "${BRANCH}" "origin/${BRANCH}"; then + git switch --orphan "${BRANCH}" + fi + touch .nojekyll + cp ../.asf.yaml ./ + git add .nojekyll .asf.yaml + - name: Download + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-html + - name: Extract + run: | + tar -xf html.tar.gz + rm -rf site/main + mv html site/main + git -C site add main + if [ "$(git status --prcelain)" == "" ]; then + NEED_PUSH=true + else + NEED_PUSH=false + fi + echo "NEED_PUSH=${NEED_PUSH}" >> ${GITHUB_ENV} + - name: Push + if: env.NEED_PUSH == 'true' + run: | + cd site + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "Publish documentation (${GITHUB_SHA})" + git push origin "${BRANCH}" + verify: + name: Verify + needs: + - binaries + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: release-* + - name: Verify + run: | + mv release-source/* ./ + tar_gz=$(echo apache-arrow-java-*.tar.gz) + version=${tar_gz#apache-arrow-java-} + version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=$(date +%Y%m%d) + fi + tar -xf ${tar_gz} + export VERIFY_DEFAULT=0 + export VERIFY_BINARY=1 + export VERIFY_SOURCE=1 + cd apache-arrow-java-${version} + mv ../${tar_gz}* ./ + mv ../release-binaries binaries + dev/release/verify_rc.sh "${version}" "${rc}" + upload: + name: Upload + if: github.ref_type == 'tag' + needs: + - docs + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: release-* + path: artifacts + - name: Upload + run: | + # GH-499: How to create release notes? + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + gh release create ${GITHUB_REF_NAME} \ + --generate-notes \ + --prerelease \ + --repo ${GITHUB_REPOSITORY} \ + --title "Apache Arrow Java ${version} RC${rc}" \ + --verify-tag + # GitHub CLI does not respect their own rate limits + # https://github.com/cli/cli/issues/9586 + for artifact in artifacts/*/*; do + sleep 1 + gh release upload ${GITHUB_REF_NAME} \ + --repo ${GITHUB_REPOSITORY} \ + $artifact + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 87bb0336078cf0edb33b0f67bd071a1a3a5488a5 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Thu, 20 Feb 2025 10:22:17 -0800 Subject: [PATCH 02/14] Rename action --- .github/workflows/jarbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 5ba080fd8a..3725045e72 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: RC +name: JarBuild on: workflow_dispatch: inputs: From 12274e20c01addbaa5c995baaef434d388c1f4b6 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Thu, 20 Feb 2025 16:08:41 -0800 Subject: [PATCH 03/14] Remove publish docs step for jar build --- .github/workflows/jarbuild.yml | 67 ---------------------------------- 1 file changed, 67 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 3725045e72..d1a8ba38cb 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -473,73 +473,6 @@ jobs: with: name: release-html path: html.tar.gz - publish-docs: - name: Publish docs - # Run only when: - # * We push to a branch - # * If the target repository is apache/arrow-java: - # * The target branch is main - # * Else (fork repositories): - # * All branches - # * We can preview the last pushed content - # at https://${YOUR_GITHUB_ACCOUNT}.github.io/arrow-java/main/ - if: >- - github.event_name == 'push' && - github.ref_type == 'branch' && - ((github.repository == 'apache/arrow-java' && github.ref_name == 'main') || - github.repository != 'apache/arrow-java') - needs: - - docs - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: site - - name: Prepare branch - run: | - if [ "${GITHUB_REPOSITORY}" = "apache/arrow-java" ]; then - BRANCH=asf-site - else - BRANCH=gh-pages - fi - echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV} - - cp site/.asf.yaml ./ - cd site - git fetch - if ! git switch -c "${BRANCH}" "origin/${BRANCH}"; then - git switch --orphan "${BRANCH}" - fi - touch .nojekyll - cp ../.asf.yaml ./ - git add .nojekyll .asf.yaml - - name: Download - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: release-html - - name: Extract - run: | - tar -xf html.tar.gz - rm -rf site/main - mv html site/main - git -C site add main - if [ "$(git status --prcelain)" == "" ]; then - NEED_PUSH=true - else - NEED_PUSH=false - fi - echo "NEED_PUSH=${NEED_PUSH}" >> ${GITHUB_ENV} - - name: Push - if: env.NEED_PUSH == 'true' - run: | - cd site - git config --global user.name 'github-actions[bot]' - git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git commit -m "Publish documentation (${GITHUB_SHA})" - git push origin "${BRANCH}" verify: name: Verify needs: From 329357d5601b013e70a584e81a94f84324eb1124 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Fri, 21 Feb 2025 17:54:59 -0800 Subject: [PATCH 04/14] release for jar build --- .github/workflows/jarbuild.yml | 42 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index d1a8ba38cb..8dcab7f968 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -22,11 +22,16 @@ on: arrow_branch: description: 'Arrow branch to build' required: true + default: main arrow_repo: description: 'Arrow repository' type: string required: true - default: https://github.com/dremio/arrow.git + default: apache/arrow + release_tag_name: + description: 'release tag name' + type: string + required: true concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -44,23 +49,14 @@ jobs: with: submodules: recursive - name: Prepare for tag - if: github.ref_type == 'tag' run: | - version=${GITHUB_REF_NAME%-rc*} + version=${release_tag_name%-rc*} version=${version#v} - rc=${GITHUB_REF_NAME#*-rc} - echo "VERSION=${version}" >> ${GITHUB_ENV} - echo "RC=${rc}" >> ${GITHUB_ENV} - - name: Prepare for branch - if: github.ref_type == 'branch' - run: | - version=$(grep -o '^ .*' "pom.xml" | - sed \ - -e 's,^ ,,' \ - -e 's,$,,') - rc=$(date +%Y%m%d) + rc=${release_tag_name#*-rc} echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "VERSION=${version}" echo "RC=${rc}" >> ${GITHUB_ENV} + echo "RC=${rc}" - name: Archive run: | id="apache-arrow-java-${VERSION}" @@ -117,7 +113,8 @@ jobs: if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: apache/arrow + repository: ${arrow_repo} + ref: ${arrow_branch} path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -184,7 +181,8 @@ jobs: if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: apache/arrow + repository: ${arrow_repo} + ref: ${arrow_branch} path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -310,7 +308,8 @@ jobs: # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: apache/arrow + repository: ${arrow_repo} + ref: ${arrow_branch} path: arrow - name: Set up Java uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 @@ -511,7 +510,6 @@ jobs: dev/release/verify_rc.sh "${version}" "${rc}" upload: name: Upload - if: github.ref_type == 'tag' needs: - docs - verify @@ -527,10 +525,10 @@ jobs: - name: Upload run: | # GH-499: How to create release notes? - version=${GITHUB_REF_NAME%-rc*} + version=${release_tag_name%-rc*} version=${version#v} - rc=${GITHUB_REF_NAME#*-rc} - gh release create ${GITHUB_REF_NAME} \ + rc=${release_tag_name#*-rc} + gh release create ${release_tag_name} \ --generate-notes \ --prerelease \ --repo ${GITHUB_REPOSITORY} \ @@ -540,7 +538,7 @@ jobs: # https://github.com/cli/cli/issues/9586 for artifact in artifacts/*/*; do sleep 1 - gh release upload ${GITHUB_REF_NAME} \ + gh release upload ${release_tag_name} \ --repo ${GITHUB_REPOSITORY} \ $artifact done From b327d3ef0f362861de319d7121982a8315c0eba8 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Fri, 21 Feb 2025 18:14:50 -0800 Subject: [PATCH 05/14] sdfsd --- .github/workflows/jarbuild.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 8dcab7f968..cc8011dfcc 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -48,8 +48,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive + - name: Set env + run: echo "release_tag_name=$(echo $release_tag_name)" >> $GITHUB_ENV - name: Prepare for tag run: | + echo "${release_tag_name}" version=${release_tag_name%-rc*} version=${version#v} rc=${release_tag_name#*-rc} From f2bf31c15a39646be83f331ca7962e213ae7fbd5 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 13:54:32 -0800 Subject: [PATCH 06/14] test --- .github/workflows/jarbuild.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index cc8011dfcc..760d2b8582 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -52,8 +52,10 @@ jobs: run: echo "release_tag_name=$(echo $release_tag_name)" >> $GITHUB_ENV - name: Prepare for tag run: | - echo "${release_tag_name}" - version=${release_tag_name%-rc*} + echo "${{release_tag_name}}" + rt = "${{release_tag_name}}" + echo "rt is ${rt}" + version=${rt%-rc*} version=${version#v} rc=${release_tag_name#*-rc} echo "VERSION=${version}" >> ${GITHUB_ENV} From 92f560ebfd613f7836d9892702247c9d5a2c6aa1 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 13:59:09 -0800 Subject: [PATCH 07/14] test --- .github/workflows/jarbuild.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 760d2b8582..e39c12a53a 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -52,8 +52,8 @@ jobs: run: echo "release_tag_name=$(echo $release_tag_name)" >> $GITHUB_ENV - name: Prepare for tag run: | - echo "${{release_tag_name}}" - rt = "${{release_tag_name}}" + echo "${{github.event.inputs.release_tag_name}}" + rt = "${{github.event.inputs.release_tag_name}}" echo "rt is ${rt}" version=${rt%-rc*} version=${version#v} From 93bb6a9dd264c01badb6529bf795e0737b78cb83 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 14:15:43 -0800 Subject: [PATCH 08/14] test --- .github/workflows/jarbuild.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index e39c12a53a..c6fcee8529 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -53,11 +53,9 @@ jobs: - name: Prepare for tag run: | echo "${{github.event.inputs.release_tag_name}}" - rt = "${{github.event.inputs.release_tag_name}}" - echo "rt is ${rt}" - version=${rt%-rc*} + version=${{github.event.inputs.release_tag_name}%-rc*} version=${version#v} - rc=${release_tag_name#*-rc} + rc=${github.event.inputs.release_tag_name#*-rc} echo "VERSION=${version}" >> ${GITHUB_ENV} echo "VERSION=${version}" echo "RC=${rc}" >> ${GITHUB_ENV} From 70984d0245b2f8c5c38bf81fc770a2bb4d7e55ef Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 14:19:51 -0800 Subject: [PATCH 09/14] test --- .github/workflows/jarbuild.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index c6fcee8529..024366feaf 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -53,9 +53,9 @@ jobs: - name: Prepare for tag run: | echo "${{github.event.inputs.release_tag_name}}" - version=${{github.event.inputs.release_tag_name}%-rc*} + version=${ ${{github.event.inputs.release_tag_name}}%-rc*} version=${version#v} - rc=${github.event.inputs.release_tag_name#*-rc} + rc=${ ${{github.event.inputs.release_tag_name}}#*-rc} echo "VERSION=${version}" >> ${GITHUB_ENV} echo "VERSION=${version}" echo "RC=${rc}" >> ${GITHUB_ENV} From 639f4f83cf1d152317dab0127345ceebd4ef19ae Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 14:55:37 -0800 Subject: [PATCH 10/14] test --- .github/workflows/jarbuild.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 024366feaf..b947431a48 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -53,9 +53,10 @@ jobs: - name: Prepare for tag run: | echo "${{github.event.inputs.release_tag_name}}" - version=${ ${{github.event.inputs.release_tag_name}}%-rc*} + ver=echo "${{github.event.inputs.release_tag_name}}" + version=${ver%-rc*} version=${version#v} - rc=${ ${{github.event.inputs.release_tag_name}}#*-rc} + rc=${ver#*-rc} echo "VERSION=${version}" >> ${GITHUB_ENV} echo "VERSION=${version}" echo "RC=${rc}" >> ${GITHUB_ENV} From 5016023247906bedd4bf2d7f09144ed212b06bee Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 14:59:41 -0800 Subject: [PATCH 11/14] test --- .github/workflows/jarbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index b947431a48..0b7badf857 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -53,7 +53,7 @@ jobs: - name: Prepare for tag run: | echo "${{github.event.inputs.release_tag_name}}" - ver=echo "${{github.event.inputs.release_tag_name}}" + ver=$(echo ${{github.event.inputs.release_tag_name}}) version=${ver%-rc*} version=${version#v} rc=${ver#*-rc} From 7934eab6da54eadc6fc7dd1dd7938065ba52078d Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 15:25:05 -0800 Subject: [PATCH 12/14] use build vars --- .github/workflows/jarbuild.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 0b7badf857..9ca3dc23c8 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -109,16 +109,16 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - run: | - ci/scripts/download_cpp.sh + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: ${arrow_repo} - ref: ${arrow_branch} + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -177,16 +177,16 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - run: | - ci/scripts/download_cpp.sh + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: ${arrow_repo} - ref: ${arrow_branch} + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -312,8 +312,8 @@ jobs: # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: ${arrow_repo} - ref: ${arrow_branch} + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} path: arrow - name: Set up Java uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 From 602fe466e2e361cadf6e7de3969665233cc82716 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 16:49:06 -0800 Subject: [PATCH 13/14] use build vars --- .github/workflows/jarbuild.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 9ca3dc23c8..292dd1a0b8 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -529,10 +529,12 @@ jobs: - name: Upload run: | # GH-499: How to create release notes? - version=${release_tag_name%-rc*} + echo "${{github.event.inputs.release_tag_name}}" + ver=$(echo ${{github.event.inputs.release_tag_name}}) + version=${ver%-rc*} version=${version#v} - rc=${release_tag_name#*-rc} - gh release create ${release_tag_name} \ + rc=${ver#*-rc} + gh release create ${{github.event.inputs.release_tag_name}} \ --generate-notes \ --prerelease \ --repo ${GITHUB_REPOSITORY} \ From 9839602eb737e3e9cbc23e64be44fa0dbfd46235 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 24 Feb 2025 17:54:33 -0800 Subject: [PATCH 14/14] use build vars --- .github/workflows/jarbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 292dd1a0b8..ae2981cd6d 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -544,7 +544,7 @@ jobs: # https://github.com/cli/cli/issues/9586 for artifact in artifacts/*/*; do sleep 1 - gh release upload ${release_tag_name} \ + gh release upload ${{github.event.inputs.release_tag_name}} \ --repo ${GITHUB_REPOSITORY} \ $artifact done