diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..849706ab659 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,178 @@ +version: 2 +updates: + # Keep version updates disabled so this config only groups security updates. + - package-ecosystem: "bundler" + directory: "/docs" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + docs-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "npm" + directory: "/zeppelin-web" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + zeppelin-web-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "npm" + directory: "/zeppelin-web-angular" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + zeppelin-web-angular-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "pip" + directory: "/dev" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + dev-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + root-maven-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/bigquery" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + bigquery-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/elasticsearch" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + elasticsearch-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/flink/flink-scala-2.12" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + flink-scala-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/livy" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + livy-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/shell" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + shell-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/spark/interpreter" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + spark-interpreter-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/spark/spark-scala-parent" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + spark-scala-parent-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/zeppelin-interpreter" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + zeppelin-interpreter-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/zeppelin-plugins/launcher/docker" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + launcher-docker-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/zeppelin-plugins/launcher/k8s-standard" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + launcher-k8s-standard-security-updates: + applies-to: security-updates + patterns: + - "*" + + - package-ecosystem: "maven" + directory: "/zeppelin-plugins/notebookrepo/s3" + schedule: + interval: "weekly" + open-pull-requests-limit: 0 + groups: + notebookrepo-s3-security-updates: + applies-to: security-updates + patterns: + - "*" diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 55a84f9b1eb..816d9f7be90 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -33,7 +33,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - # test on core-modules (zeppelin-interpreter,zeppelin-zengine,zeppelin-server), + # test on core-modules (zeppelin-interpreter,zeppelin-server), # some interpreters are included, because zeppelin-server test depends on them: spark, shell & markdown core-modules: runs-on: ubuntu-24.04 @@ -43,16 +43,16 @@ jobs: java: [ 11 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -67,25 +67,24 @@ jobs: run: ./mvnw install -Pbuild-distr -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples ${MAVEN_ARGS} - name: install and test plugins run: ./mvnw package -pl zeppelin-plugins -amd ${MAVEN_ARGS} - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_3.9_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_3.9.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter + - name: Show conda environment run: | - R -e "IRkernel::installspec()" conda list conda info - name: run tests # skip spark test because we would run them in other CI run: ./mvnw verify -Pusing-packaged-distr -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false - # test interpreter modules except spark, flink, python, rlang, jupyter + # test interpreter modules except spark, flink, python, jupyter interpreter-test-non-core: runs-on: ubuntu-24.04 strategy: @@ -93,19 +92,19 @@ jobs: matrix: java: [ 11 ] env: - INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' + INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -117,21 +116,21 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: ./mvnw install -DskipTests -am -pl ${INTERPRETERS} ${MAVEN_ARGS} - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R_and_tensorflow - environment-file: testing/env_python_3_with_R_and_tensorflow.yml + activate-environment: python_3_with_tensorflow + environment-file: testing/env_python_3_with_tensorflow.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - name: verify interpreter run: ./mvnw verify -am -pl ${INTERPRETERS} ${MAVEN_ARGS} - # test interpreter modules for jupyter, python, rlang - interpreter-test-jupyter-python-rlang: + # test interpreter modules for jupyter, python + interpreter-test-jupyter-python: runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -140,16 +139,16 @@ jobs: java: [ 11 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -159,25 +158,22 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python ${{ matrix.python }} and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python ${{ matrix.python }} + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_${{ matrix.python }}.yml python-version: ${{ matrix.python }} channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: install environment run: | - ./mvnw install -DskipTests -pl python,rlang,zeppelin-jupyter-interpreter -am ${MAVEN_ARGS} + ./mvnw install -DskipTests -pl python,zeppelin-jupyter-interpreter -am ${MAVEN_ARGS} - name: run tests with ${{ matrix.python }} run: | - ./mvnw test -pl python,rlang,zeppelin-jupyter-interpreter -DfailIfNoTests=false ${MAVEN_ARGS} + ./mvnw test -pl python,zeppelin-jupyter-interpreter -DfailIfNoTests=false ${MAVEN_ARGS} # zeppelin integration test except Spark & Flink zeppelin-integration-test: @@ -191,16 +187,16 @@ jobs: - name: Start mysql run: sudo systemctl start mysql.service - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -212,21 +208,18 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: | - ./mvnw install -DskipTests -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pweb-classic -Pflink-117 ${MAVEN_ARGS} + ./mvnw install -DskipTests -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pweb-classic -Pflink-1.20 ${MAVEN_ARGS} ./mvnw package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_3.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: run tests run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest - name: Print zeppelin logs @@ -238,25 +231,25 @@ jobs: strategy: fail-fast: false matrix: - python: [ 3.9 ] - flink: [116, 117] include: - # Flink 1.15 supports Python 3.6, 3.7, and 3.8 - # https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/python/installation/ - - python: 3.8 - flink: 115 + - python: 3.9 + flink: 119 + flink-profile: "1.19" + - python: 3.9 + flink: 120 + flink-profile: "1.20" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Set up JDK 8 - uses: actions/setup-java@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -268,20 +261,20 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment for flink run: | - ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration ${MAVEN_ARGS} + ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink-profile }} -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - - name: Setup conda environment with python ${{ matrix.python }} and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python ${{ matrix.python }} + uses: conda-incubator/setup-miniconda@v4 with: activate-environment: python_3_with_flink environment-file: testing/env_python_3_with_flink_${{ matrix.flink }}.yml python-version: ${{ matrix.python }} channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - name: run tests for flink - run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} + run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink-profile }} -am -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} - name: Print zeppelin logs if: always() run: if [ -d "logs" ]; then cat logs/*; fi @@ -295,16 +288,16 @@ jobs: java: [ 11 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -318,19 +311,16 @@ jobs: run: | ./mvnw install -DskipTests -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown -am -Pweb-classic -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_3.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: run tests run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest32,SparkIntegrationTest32,ZeppelinSparkClusterTest33,SparkIntegrationTest33 -DfailIfNoTests=false ${MAVEN_ARGS} @@ -344,16 +334,16 @@ jobs: java: [ 11, 17 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -365,19 +355,16 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am ${MAVEN_ARGS} - - name: Setup conda environment with python ${{ matrix.python }} and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python ${{ matrix.python }} + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_${{ matrix.python }}.yml python-version: ${{ matrix.python }} channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} if: ${{ matrix.java == 11 }} run: | @@ -410,16 +397,16 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -434,19 +421,16 @@ jobs: ./mvnw install -DskipTests -pl livy -am ${MAVEN_ARGS} ./testing/downloadSpark.sh "3.2.4" "3.2" ./testing/downloadLivy.sh "0.8.0-incubating" "2.12" - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_39_with_R - environment-file: testing/env_python_3.9_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_3.9.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: run tests run: | export SPARK_HOME=$PWD/spark-3.2.4-bin-hadoop3.2 @@ -461,16 +445,16 @@ jobs: java: [ 11 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -491,14 +475,14 @@ jobs: java: [ 11 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~\.m2\repository diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..f9d77459a74 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,62 @@ +name: docs + +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'docs/**' + - '.github/workflows/docs.yml' + pull_request: + branches: + - master + - 'branch-*' + paths: + - 'docs/**' + - '.github/workflows/docs.yml' + +permissions: + contents: read + +jobs: + internal-link-check: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + bundler-cache: true + working-directory: docs + - name: Build the Jekyll site + working-directory: docs + run: bundle exec jekyll build --safe -d _site + - name: Check internal links + working-directory: docs + # Report-only: broken links are printed in the log but do not fail the + # build yet. Remove this once the job has been green for a while, so + # that broken links start blocking merges (ZEPPELIN-6628). + continue-on-error: true + run: | + gem install html-proofer -v 5.2.2 --no-document + # Template links are prefixed with JB.BASE_PATH (e.g. + # /docs/0.13.0-SNAPSHOT), a prefix that only exists once the site is + # published. Strip it so links resolve against the built tree. Read it + # from _config.yml so a version bump does not break this job. + BASE_PATH=$(ruby -ryaml -e 'puts(YAML.load_file("_config.yml")["JB"]["BASE_PATH"] || "")') + echo "Stripping BASE_PATH prefix: ${BASE_PATH}" + # Only internal links are in scope. External URLs are skipped because + # they break for reasons outside this repository and would make the + # job flaky; anchor fragments are skipped for the same reason. The + # --allow-missing-href flag keeps `` anchors, which the + # docs use as link targets, from being reported as errors. + htmlproofer _site \ + --root-dir _site \ + --checks Links \ + --disable-external \ + --no-enforce-https \ + --no-check-internal-hash \ + --allow-missing-href \ + --swap-urls "^${BASE_PATH}:" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 2f99846e696..1998142a41d 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -23,43 +23,26 @@ env: SPARK_PRINT_LAUNCH_COMMAND: "true" SPARK_LOCAL_IP: 127.0.0.1 ZEPPELIN_LOCAL_IP: 127.0.0.1 - INTERPRETERS: '!hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!alluxio,!livy,!groovy,!java,!neo4j,!sparql,!mongodb' + INTERPRETERS: '!hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!livy,!groovy,!java,!neo4j,!sparql,!mongodb' ZEPPELIN_E2E_TEST_NOTEBOOK_DIR: '/tmp/zeppelin-e2e-notebooks' permissions: contents: read # to fetch code (actions/checkout) jobs: - run-e2e-tests-in-zeppelin-web: + npm-audit: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 11 - - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/checkout@v5 + - name: Set up Node.js + uses: actions/setup-node@v5 with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/zeppelin/ - ~/.spark-dist - ~/.cache - key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-zeppelin- - - name: Install application - run: ./mvnw clean install -DskipTests -am -pl zeppelin-web,zeppelin-web-angular -Pweb-classic -Pspark-scala-2.12 -Pspark-3.4 -Pweb-dist ${MAVEN_ARGS} - - name: Run headless test - run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -pl zeppelin-web -Pweb-classic -Pspark-scala-2.12 -Pspark-3.4 -Pweb-dist -Pweb-e2e ${MAVEN_ARGS} - - name: Print zeppelin logs - if: always() - run: if [ -d "logs" ]; then cat logs/*; fi + node-version-file: 'zeppelin-web-angular/.nvmrc' + # TODO: Add zeppelin-web-angular root audit after Angular version upgrade and stabilization + - name: Run npm audit on zeppelin-react + working-directory: zeppelin-web-angular/projects/zeppelin-react + run: npm ci --ignore-scripts && npm audit --audit-level=high run-playwright-e2e-tests: runs-on: ubuntu-24.04 @@ -72,23 +55,23 @@ jobs: python: [ 3.9 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('zeppelin-web-angular/package-lock.json') }} restore-keys: | ${{ runner.os }}-playwright- - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -99,16 +82,20 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: Setup conda environment with python ${{ matrix.python }} - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@v4 with: activate-environment: python_only python-version: ${{ matrix.python }} - auto-activate-base: false + auto-activate: false use-mamba: true channels: conda-forge,defaults channel-priority: strict - name: Install application - run: ./mvnw clean install -DskipTests -am -pl python,rlang,zeppelin-jupyter-interpreter,zeppelin-web-angular ${MAVEN_ARGS} + run: ./mvnw clean install -DskipTests -am -pl python,zeppelin-jupyter-interpreter,zeppelin-web,zeppelin-web-angular -Pweb-classic ${MAVEN_ARGS} + # Keeps the Karma coverage previously run by the removed zeppelin-web e2e job + - name: Run zeppelin-web unit tests + if: matrix.mode == 'anonymous' + run: ./mvnw test -pl zeppelin-web -Pweb-classic,web-unit-test ${MAVEN_ARGS} - name: Setup Zeppelin Server (Shiro.ini) run: | export ZEPPELIN_CONF_DIR=./conf @@ -122,13 +109,16 @@ jobs: mkdir -p $ZEPPELIN_E2E_TEST_NOTEBOOK_DIR echo "Created test notebook directory: $ZEPPELIN_E2E_TEST_NOTEBOOK_DIR" - name: Run headless E2E test with Maven - run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -pl zeppelin-web-angular -Pweb-e2e ${MAVEN_ARGS} + # Classic UI e2e runs only on the anonymous leg, like the legacy Protractor suite + run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -pl zeppelin-web-angular -Pweb-e2e -Dweb.e2e.classic.disabled=${{ matrix.mode != 'anonymous' }} ${MAVEN_ARGS} - name: Upload Playwright Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 if: always() with: name: playwright-report-${{ matrix.mode }} - path: zeppelin-web-angular/playwright-report/ + path: | + zeppelin-web-angular/playwright-report/ + zeppelin-web-angular/playwright-report-classic/ retention-days: 3 - name: Print Zeppelin logs if: always() @@ -149,36 +139,18 @@ jobs: defaults: run: shell: bash -l {0} - env: - ZEPPELIN_SELENIUM_BROWSER: "edge" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Install Microsoft Edge - run: | - curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-edge.gpg - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-edge.gpg] https://packages.microsoft.com/repos/edge stable main" | sudo tee /etc/apt/sources.list.d/microsoft-edge.list - sudo apt-get update - sudo apt-get install -y microsoft-edge-stable - - name: Install msedgedriver - run: | - EDGE_VERSION=$(microsoft-edge --version | awk '{print $3}') - wget -q "https://msedgedriver.microsoft.com/${EDGE_VERSION}/edgedriver_linux64.zip" -O edgedriver.zip - unzip -q edgedriver.zip - sudo mv msedgedriver /usr/local/bin/ - sudo chmod +x /usr/local/bin/msedgedriver - rm edgedriver.zip - - name: Print Edge version - run: msedgedriver --version - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 - name: Cache local Maven repository - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.m2/repository @@ -188,19 +160,16 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python 3.9 and R - uses: conda-incubator/setup-miniconda@v3 + - name: Setup conda environment with python 3.9 + uses: conda-incubator/setup-miniconda@v4 with: - activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml + activate-environment: python_3 + environment-file: testing/env_python_3.yml python-version: 3.9 channels: conda-forge,defaults channel-priority: strict - auto-activate-base: false + auto-activate: false use-mamba: true - - name: Make IRkernel available to Jupyter - run: | - R -e "IRkernel::installspec()" - name: Install Environment run: | ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.5 -Pweb-dist ${MAVEN_ARGS} diff --git a/.github/workflows/npm-audit-remediation.yml b/.github/workflows/npm-audit-remediation.yml new file mode 100644 index 00000000000..841bf3aab2a --- /dev/null +++ b/.github/workflows/npm-audit-remediation.yml @@ -0,0 +1,291 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: npm audit remediation + +on: + schedule: + - cron: '23 2 * * *' + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/npm-audit-remediation.yml' + +concurrency: + group: npm-audit-remediation + cancel-in-progress: false + +env: + FRONTEND_DIRECTORY: zeppelin-web-angular + PACKAGE_DIRECTORY: zeppelin-web-angular/projects/zeppelin-react + LOCKFILE: zeppelin-web-angular/projects/zeppelin-react/package-lock.json + REMEDIATION_BRANCH: automation/npm-audit-fix-zeppelin-react + PR_TITLE: '[HOTFIX] Refresh zeppelin-react lockfile for npm audit' + +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + outputs: + audited_sha: ${{ steps.revision.outputs.sha }} + needs_remediation: ${{ steps.audit.outputs.needs_remediation }} + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }} + + - id: revision + name: Record audited revision + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version-file: 'zeppelin-web-angular/.nvmrc' + + - id: audit + name: Check npm audit + working-directory: ${{ env.PACKAGE_DIRECTORY }} + shell: bash + run: | + set +e + npm audit --package-lock-only --audit-level=high --json \ + > "${RUNNER_TEMP}/npm-audit-before.json" + audit_status=$? + set -e + + if [[ ${audit_status} -eq 0 ]]; then + echo "needs_remediation=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + vulnerable_count=$(node - "${RUNNER_TEMP}/npm-audit-before.json" <<'NODE' + const fs = require('fs'); + const report = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); + const counts = report.metadata && report.metadata.vulnerabilities; + if (!counts) { + process.exit(2); + } + console.log((counts.high || 0) + (counts.critical || 0)); + NODE + ) || { + cat "${RUNNER_TEMP}/npm-audit-before.json" + echo "::error::npm audit failed without a valid vulnerability report" + exit "${audit_status}" + } + + if [[ ${vulnerable_count} -eq 0 ]]; then + cat "${RUNNER_TEMP}/npm-audit-before.json" + echo "::error::npm audit failed without a high or critical vulnerability" + exit "${audit_status}" + fi + + echo "needs_remediation=true" >> "${GITHUB_OUTPUT}" + echo "Found ${vulnerable_count} high or critical vulnerabilities" + + - name: Generate a lockfile-only fix + if: steps.audit.outputs.needs_remediation == 'true' + working-directory: ${{ env.PACKAGE_DIRECTORY }} + run: npm audit fix --package-lock-only --ignore-scripts --audit-level=high + + - name: Verify the generated diff + if: steps.audit.outputs.needs_remediation == 'true' + shell: bash + run: | + changed_files=$(git diff --name-only) + if [[ "${changed_files}" != "${LOCKFILE}" ]]; then + echo "::error::Expected only ${LOCKFILE} to change, got:" + printf '%s\n' "${changed_files}" + exit 1 + fi + git diff --check + + - name: Install frontend dependencies + if: steps.audit.outputs.needs_remediation == 'true' + working-directory: ${{ env.FRONTEND_DIRECTORY }} + run: npm ci --ignore-scripts --no-audit + + - name: Validate the fix + if: steps.audit.outputs.needs_remediation == 'true' + working-directory: ${{ env.PACKAGE_DIRECTORY }} + run: | + npm ci --ignore-scripts --no-audit + npm audit --audit-level=high + npm run lint + npm test + npm run build + + - name: Create remediation artifact + if: steps.audit.outputs.needs_remediation == 'true' + shell: bash + run: | + artifact_directory="${RUNNER_TEMP}/npm-audit-remediation" + mkdir -p "${artifact_directory}" + git diff --binary -- "${LOCKFILE}" > "${artifact_directory}/fix.patch" + cp "${RUNNER_TEMP}/npm-audit-before.json" "${artifact_directory}/audit-before.json" + test -s "${artifact_directory}/fix.patch" + + - name: Upload remediation artifact + if: steps.audit.outputs.needs_remediation == 'true' + uses: actions/upload-artifact@v6 + with: + name: npm-audit-remediation + path: ${{ runner.temp }}/npm-audit-remediation + retention-days: 1 + + publish: + needs: prepare + if: >- + needs.prepare.outputs.needs_remediation == 'true' && + github.event_name != 'pull_request' && + github.repository == 'apache/zeppelin' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: write + pull-requests: write + steps: + - name: Checkout audited revision + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ needs.prepare.outputs.audited_sha }} + + - name: Download remediation artifact + uses: actions/download-artifact@v7 + with: + name: npm-audit-remediation + path: ${{ runner.temp }}/npm-audit-remediation + + - name: Create or update remediation pull request + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + git apply "${RUNNER_TEMP}/npm-audit-remediation/fix.patch" + + changed_files=$(git diff --name-only) + if [[ "${changed_files}" != "${LOCKFILE}" ]]; then + echo "::error::Artifact changed unexpected files:" + printf '%s\n' "${changed_files}" + exit 1 + fi + + desired_blob=$(git hash-object "${LOCKFILE}") + # Include closed PRs so a maintainer's decision is not undone every day. + # The REST head filter includes the repository owner, avoiding a fork + # PR with the same predictable branch name. + latest_pr=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/pulls" \ + -f state=all \ + -f base=master \ + -f head="${GITHUB_REPOSITORY_OWNER}:${REMEDIATION_BRANCH}" \ + -f per_page=1 \ + --jq '.[0] // empty | {number: .number, state: (.state | ascii_upcase)}') + + latest_pr_number=$(jq -r '.number // empty' <<< "${latest_pr}") + latest_pr_state=$(jq -r '.state // empty' <<< "${latest_pr}") + + if [[ -n "${latest_pr_number}" ]]; then + unexpected_files=$(gh pr view "${latest_pr_number}" \ + --repo "${GITHUB_REPOSITORY}" \ + --json files \ + --jq '.files[].path' | grep -vx "${LOCKFILE}" || true) + + if [[ "${latest_pr_state}" == "OPEN" && -n "${unexpected_files}" ]]; then + echo "::error::Existing remediation PR contains unexpected files:" + printf '%s\n' "${unexpected_files}" + exit 1 + fi + + git fetch --no-tags origin \ + "refs/pull/${latest_pr_number}/head:refs/remotes/origin/npm-audit-pr-head" + previous_blob=$(git rev-parse \ + "refs/remotes/origin/npm-audit-pr-head:${LOCKFILE}" 2>/dev/null || true) + + if [[ -z "${unexpected_files}" && "${desired_blob}" == "${previous_blob}" ]]; then + echo "PR #${latest_pr_number} (${latest_pr_state}) already contains this lockfile." + echo "Suppressing an identical replacement PR." + exit 0 + fi + + if [[ "${latest_pr_state}" == "OPEN" ]]; then + open_pr_number="${latest_pr_number}" + fi + fi + + remote_sha=$(git ls-remote --heads origin \ + "refs/heads/${REMEDIATION_BRANCH}" | cut -f1) + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -C "${REMEDIATION_BRANCH}" + git add "${LOCKFILE}" + git commit -m "${PR_TITLE}" + + if [[ -n "${remote_sha}" ]]; then + git push \ + --force-with-lease="refs/heads/${REMEDIATION_BRANCH}:${remote_sha}" \ + origin "HEAD:refs/heads/${REMEDIATION_BRANCH}" + else + git push origin "HEAD:refs/heads/${REMEDIATION_BRANCH}" + fi + + if [[ -n "${open_pr_number:-}" ]]; then + echo "Updated existing PR #${open_pr_number}." + exit 0 + fi + + pr_body="${RUNNER_TEMP}/npm-audit-pr-body.md" + { + echo '### What is this PR for?' + echo + echo 'Refresh the zeppelin-react lockfile with compatible updates suggested by `npm audit fix`.' + echo 'The daily audit remediation workflow generated this change after the required high-severity audit began failing.' + echo + echo '### What type of PR is it?' + echo + echo 'Hot Fix' + echo + echo '### What is the Jira issue?' + echo + echo 'N/A - automated dependency maintenance.' + echo + echo '### How should this be tested?' + echo + echo '- `npm ci --ignore-scripts --no-audit` in `zeppelin-web-angular`' + echo '- `npm ci --ignore-scripts --no-audit` in `zeppelin-web-angular/projects/zeppelin-react`' + echo '- `npm audit --audit-level=high`' + echo '- `npm run lint`' + echo '- `npm test`' + echo '- `npm run build`' + echo + echo 'All commands passed before this PR was created.' + } > "${pr_body}" + + gh pr create \ + --repo "${GITHUB_REPOSITORY}" \ + --base master \ + --head "${REMEDIATION_BRANCH}" \ + --title "${PR_TITLE}" \ + --body-file "${pr_body}" \ + --draft diff --git a/.github/workflows/quick.yml b/.github/workflows/quick.yml index 27c9493daff..6e2f7029b75 100644 --- a/.github/workflows/quick.yml +++ b/.github/workflows/quick.yml @@ -28,9 +28,9 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 @@ -42,9 +42,9 @@ jobs: fail-fast: false steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 11 diff --git a/.gitignore b/.gitignore index a77b3b5db6f..25319152ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,25 @@ tramp # dotenv files .env + +# AI coding agents — personal config (AGENTS.md is shared, these are not) +CLAUDE.md +GEMINI.md +.claude/ +.gemini/ +.codex/ +.cursor/ +.cursorules +.cursorrules +.windsurf/ +.windsurfrules +.cline/ +.clinerules +.continue/ +.aider* +.augment/ +.amazonq/ +.junie/ +.goose/ +.roo/ +.rooignore diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..090dc4433e1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,572 @@ + + +# AGENTS.md + +> Guidance for AI coding agents working on the Apache Zeppelin codebase. +> See [AGENTS.md specification](https://github.com/agentsmd/agents.md). + +## Project Overview + +Apache Zeppelin is a web-based notebook for interactive data analytics. It provides a unified interface to multiple data processing backends (Spark, Flink, Python, JDBC, etc.) through a pluggable interpreter architecture. Each interpreter runs in its own JVM process and communicates with the server via Apache Thrift RPC. + +- **Language**: Java, Scala — versions in root `pom.xml` (`java.version`, `scala.binary.version`) +- **Build**: Maven multi-module (wrapper: `./mvnw`) +- **Frontend**: Angular + TypeScript in `zeppelin-web-angular/` — versions in its `package.json` +- **Version**: see `` in root `pom.xml` + +## Build & Test + +```bash +# Full build (skip tests) +./mvnw clean package -DskipTests + +# Build single module (--am builds required upstream modules) +./mvnw clean package -pl zeppelin-server --am -DskipTests + +# Run module tests +./mvnw test -pl zeppelin-interpreter --am + +# Run single test class/method +./mvnw test -pl zeppelin-server --am -Dtest=NotebookServerTest +./mvnw test -pl zeppelin-server --am -Dtest=NotebookServerTest#testMethod + +# Common profiles +# -Pspark-3.5 -Pspark-scala-2.12 Spark version +# -Pflink-1.20 Flink version +# -Pbuild-distr Full distribution +# -Prat Apache RAT license check +# -Pweb-classic Additionally builds the classic UI web module when specified +``` + +## Build Gotchas + +### Shaded JAR Rebuild Chain + +The most common build mistake: modifying `zeppelin-interpreter` without rebuilding `zeppelin-interpreter-shaded`. The shaded JAR is an uber JAR that all interpreter processes use. If it's stale, you get `ClassNotFoundException` or `NoSuchMethodError` at runtime. + +```bash +# After changing zeppelin-interpreter, ALWAYS rebuild in order: +./mvnw clean package -pl zeppelin-interpreter -DskipTests +./mvnw clean package -pl zeppelin-interpreter-shaded -DskipTests +# Then rebuild affected interpreter modules + +# Shorthand: +./mvnw clean package -pl zeppelin-interpreter,zeppelin-interpreter-shaded -DskipTests +``` + +The shaded JAR is also copied to `interpreter/` directory by maven-antrun-plugin after packaging. If this directory has a stale JAR, interpreter processes will load old code. + +### Module Build Order + +Maven modules are ordered in the root `pom.xml`. Key sequence: +``` +zeppelin-interpreter → zeppelin-interpreter-shaded → zeppelin-server +``` + +All interpreter modules build after `zeppelin-interpreter-shaded`. A second shading chain exists for Jupyter: +``` +zeppelin-jupyter-interpreter → zeppelin-jupyter-interpreter-shaded → python +``` + +## Module Architecture + +### Dependency Flow + +``` +zeppelin-interpreter Base API: Interpreter, InterpreterContext, Thrift services + ↓ +zeppelin-interpreter-shaded Uber JAR (maven-shade-plugin, relocated packages) + ↓ +zeppelin-server Core engine + Jetty 11, REST/WebSocket APIs, HK2 DI, entry point +``` + +### Core Modules + +#### `zeppelin-interpreter/` +The base framework that all interpreters depend on. Defines the interpreter API and the Thrift communication protocol. This module is shaded into an uber JAR (`zeppelin-interpreter-shaded`) and placed on each interpreter process's classpath. + +Key classes: +- `Interpreter` (abstract) / `AbstractInterpreter` — base class every interpreter extends +- `InterpreterContext` — carries notebook/paragraph/user info into `interpret()` calls +- `InterpreterGroup` — manages a group of interpreter instances sharing one process +- `InterpreterResult` / `InterpreterOutput` — execution result model +- `RemoteInterpreterServer` — **entry point of each interpreter JVM process**; implements the Thrift `RemoteInterpreterService` server; receives RPC calls from zeppelin-server +- `InterpreterLauncher` (abstract) — how an interpreter process is started (Standard, Docker, K8s, YARN) +- `LifecycleManager` — manages interpreter process lifecycle (Null = keep alive, Timeout = idle shutdown) +- `DependencyResolver` / `AbstractDependencyResolver` — Maven artifact resolution for `%dep` paragraphs + +Thrift definitions (`src/main/thrift/`): +- `RemoteInterpreterService.thrift` — server → interpreter RPCs +- `RemoteInterpreterEventService.thrift` — interpreter → server event callbacks + +#### `zeppelin-server/` +The entry point and core of the Zeppelin application. Combines the web server / API layer with the core notebook engine, interpreter lifecycle management, scheduling, search, and plugin loading. + +Web / API layer (`org.apache.zeppelin.server`, `rest`, `socket`): +- `ZeppelinServer` — `main()`, embedded Jetty 11 server, HK2 DI setup +- `NotebookRestApi`, `InterpreterRestApi`, `SecurityRestApi`, `ConfigurationsRestApi` — REST endpoints in `org.apache.zeppelin.rest` +- `NotebookServer` — WebSocket endpoint (`/ws`) for real-time notebook operations and paragraph execution +- `RemoteInterpreterEventServer` — Thrift server receiving callbacks from interpreter processes (output streaming, status updates) + +Engine / runtime (`org.apache.zeppelin.notebook`, `interpreter`, `scheduler`, `search`, `plugin`, `storage`, `conf`): +- `Notebook` / `Note` / `Paragraph` — notebook data model and execution +- `InterpreterFactory` — creates interpreter instances +- `InterpreterSettingManager` — loads `interpreter-setting.json` from each interpreter directory, manages interpreter configurations +- `InterpreterSetting` — one interpreter's config + runtime state; creates `InterpreterLauncher` and `RemoteInterpreterProcess` +- `ManagedInterpreterGroup` — server-side `InterpreterGroup` implementation; owns the `RemoteInterpreterProcess` +- `NoteManager` — notebook CRUD, folder tree +- `SchedulerService` — Quartz-based cron scheduling +- `SearchService` — Lucene-based notebook search +- `PluginManager` — loads launcher and notebook-repo plugins (custom classloading, not Java SPI) +- `ZeppelinConfiguration` — config management (env vars → system properties → `zeppelin-site.xml` → defaults) +- `RecoveryStorage` — persists interpreter process info for server-restart recovery +- `ConfigStorage` — persists interpreter settings to JSON + +#### `zeppelin-interpreter-shaded/` +Uses maven-shade-plugin to package `zeppelin-interpreter` + dependencies into an uber JAR with relocated packages (e.g., `org.apache.thrift` → `org.apache.zeppelin.shaded.org.apache.thrift`). This JAR is placed on each interpreter process's classpath. + +#### `zeppelin-client/` +REST/WebSocket client library for programmatic access to Zeppelin. + +### Interpreter Modules + +Each interpreter is an independent Maven module inheriting from `zeppelin-interpreter-parent`: + +| Module | Description | +|--------|-------------| +| `spark/` | Apache Spark (Scala/Python/R/SQL) — most complex interpreter | +| `python/` | IPython/Python | +| `flink/` | Apache Flink (Scala/Python/SQL) | +| `jdbc/` | JDBC (PostgreSQL, MySQL, Hive, etc.) | +| `shell/` | Bash/Shell commands | +| `markdown/` | Markdown rendering (Flexmark) | +| `java/` | Java interpreter | +| `groovy/` | Groovy | +| `neo4j/` | Neo4j Cypher | +| `mongodb/` | MongoDB | +| `elasticsearch/` | Elasticsearch | +| `bigquery/` | Google BigQuery | +| `cassandra/` | Apache Cassandra CQL | +| `hbase/` | Apache HBase | +| `livy/` | Apache Livy (remote Spark) | +| `sparql/` | SPARQL queries | +| `influxdb/` | InfluxDB | +| `file/` | HDFS/local file browser | + +### Plugin Modules (`zeppelin-plugins/`) + +**Launcher plugins** (`launcher/`) — how interpreter processes are started: +- `StandardInterpreterLauncher` (builtin) — local JVM process via `bin/interpreter.sh` +- `SparkInterpreterLauncher` (builtin) — Spark-specific launcher with `spark-submit` +- `DockerInterpreterLauncher` — Docker container +- `K8sStandardInterpreterLauncher` — Kubernetes pod +- `YarnInterpreterLauncher` — YARN container +- `FlinkInterpreterLauncher` — Flink-specific +- `ClusterInterpreterLauncher` — Zeppelin cluster mode + +**NotebookRepo plugins** (`notebookrepo/`) — where notebooks are persisted: +- `VFSNotebookRepo` (builtin) — local filesystem (Apache VFS) +- `GitNotebookRepo` (builtin) — local git repo +- `GitHubNotebookRepo` — GitHub +- `S3NotebookRepo` — Amazon S3 +- `GCSNotebookRepo` — Google Cloud Storage +- `AzureNotebookRepo` — Azure Blob Storage +- `MongoNotebookRepo` — MongoDB +- `OSSNotebookRepo` — Alibaba Cloud OSS + +### Frontend + +- `zeppelin-web-angular/` — active frontend (Angular; versions in `package.json`, Node build pin in `pom.xml` `node.version`) +- `zeppelin-web/` — Legacy AngularJS (activated with `-Pweb-classic`) + +### Configuration Files + +| File | Purpose | +|------|---------| +| `conf/zeppelin-site.xml` | Main server config (port, SSL, notebook storage, interpreter settings). Copy from `.template` | +| `conf/zeppelin-env.sh` | Shell environment (JAVA_OPTS, memory, Spark master). Copy from `.template` | +| `conf/shiro.ini` | Authentication/authorization (users, roles, LDAP, Kerberos, PAM). Copy from `.template` | +| `conf/interpreter.json` | Runtime interpreter settings — **auto-generated**, do not edit manually | +| `conf/log4j2.properties` | Logging configuration | +| `conf/interpreter-list` | Static list of available interpreters with Maven coordinates | +| `{interpreter}/resources/interpreter-setting.json` | Interpreter defaults (build-time, bundled in JAR) | + +`conf/*.template` files are the source of truth. Actual config files (`zeppelin-site.xml`, `shiro.ini`, etc.) are `.gitignored`. + +### Module Boundaries + +Where new code should go: + +| If the code... | Put it in | +|----------------|-----------| +| Is a base interface/class that all interpreters need | `zeppelin-interpreter` | +| Handles notebook state, interpreter lifecycle, scheduling, search, REST/WebSocket, or authentication realm | `zeppelin-server` | +| Is specific to one backend (Spark, Flink, JDBC, etc.) | That interpreter's module | +| Is a new way to launch interpreter processes | `zeppelin-plugins/launcher/` | +| Is a new notebook storage backend | `zeppelin-plugins/notebookrepo/` | + +**Important**: Code added to `zeppelin-interpreter` is exposed to **every interpreter process** via the shaded JAR. Only add code there if all interpreters genuinely need it. + +## Server–Interpreter Communication + +Zeppelin's most important architectural concept: the server and each interpreter run in **separate JVM processes** communicating via **Apache Thrift RPC**. This provides isolation, fault tolerance, and the ability to run interpreters on remote hosts or containers. + +### Thrift Code Generation + +The `.thrift` files are in `zeppelin-interpreter/src/main/thrift/`. Generated Java files are **checked into git** (not generated at build time) in `zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/`. + +To regenerate after modifying `.thrift` files: +```bash +cd zeppelin-interpreter/src/main/thrift +./genthrift.sh # requires 'thrift' compiler (v0.13.0) installed locally +``` + +The script runs the Thrift compiler, prepends ASF license headers, and moves files to the source tree. **Never edit the generated Java files directly** — changes will be lost on next regeneration. + +### Thrift IPC — Bidirectional + +**Server → Interpreter** (`RemoteInterpreterService`): +``` +init(properties) — initialize interpreter process with config +createInterpreter(className, ...) — instantiate an interpreter class +open(sessionId, className) — open/initialize an interpreter +interpret(sessionId, className, code, context) — execute code (core method) +cancel(sessionId, className, ...) — cancel running execution +getProgress(sessionId, className) — poll execution progress (0-100) +completion(sessionId, className, buf, cursor) — code completion +close(sessionId, className) — close an interpreter +shutdown() — terminate the interpreter process +``` + +**Interpreter → Server** (`RemoteInterpreterEventService`): +``` +registerInterpreterProcess(info) — register after process startup +appendOutput(event) — stream execution output incrementally +updateOutput(event) — replace output content +sendParagraphInfo(info) — update paragraph metadata +updateAppStatus(event) — Zeppelin Application status +runParagraphs(request) — trigger paragraph execution from interpreter +getResource(resourceId) — access ResourcePool shared state +getParagraphList(noteId) — query notebook structure +``` + +### Paragraph Execution Chain + +When a user runs a paragraph, the full call chain is: + +``` +User clicks "Run" in browser + → WebSocket message to NotebookServer + → NotebookServer.runParagraph() + → Notebook.run() + → Paragraph.execute() + → RemoteInterpreter.interpret(code, context) + → RemoteInterpreterProcess.callRemoteFunction() + → [Thrift RPC over TCP] + → RemoteInterpreterServer.interpret() + → actual Interpreter.interpret() (e.g. SparkInterpreter) + → result returned via Thrift + → meanwhile: interpreter calls appendOutput() to stream partial results back +``` + +### Interpreter Launch Chain + +When an interpreter process needs to be started: + +``` +RemoteInterpreter.interpret() [first call triggers launch] + → ManagedInterpreterGroup.getOrCreateInterpreterProcess() + → InterpreterSetting.createInterpreterProcess() + → InterpreterSetting.createLauncher(properties) + → PluginManager.loadInterpreterLauncher(launcherPlugin) + → [builtin: Class.forName() / external: URLClassLoader] + → InterpreterLauncher.launch(context) + → new ExecRemoteInterpreterProcess(...) + → ExecRemoteInterpreterProcess.start() + → ProcessBuilder → "bin/interpreter.sh" + → java -cp ... RemoteInterpreterServer [new JVM] + → RemoteInterpreterServer.main() + → registerInterpreterProcess() callback to server +``` + +### Interpreter Process Lifecycle + +1. **Launch**: Server creates `RemoteInterpreterProcess` via launcher plugin +2. **Start**: Process starts as separate JVM (`bin/interpreter.sh` → `RemoteInterpreterServer.main()`) +3. **Register**: Process calls `registerInterpreterProcess()` back to server's `RemoteInterpreterEventServer` +4. **Init**: Server calls `init(properties)` — passes all configuration as a flat `Map` +5. **Create**: Server calls `createInterpreter(className, properties)` — instantiates interpreter via reflection +6. **Open**: First `interpret()` triggers `LazyOpenInterpreter.open()` — interpreter initializes resources +7. **Execute**: `interpret(code, context)` — runs code; partial output streams via `appendOutput()` events +8. **Shutdown**: `close()` → `shutdown()` → JVM exits +9. **Recovery**: `RecoveryStorage` persists process info; on server restart, reconnects to surviving processes + +### InterpreterGroup Scoping + +`InterpreterOption` controls process isolation via `perNote` and `perUser` settings: + +| perNote | perUser | Behavior | +|---------|---------|----------| +| `shared` | `shared` | All users share one process (default) | +| `scoped` | `shared` | Separate interpreter instance per note, same process | +| `isolated` | `shared` | Separate process per note | +| `shared` | `scoped` | Separate interpreter instance per user, same process | +| `shared` | `isolated` | Separate process per user | +| `scoped` | `scoped` | Separate instance per user+note | +| `isolated` | `isolated` | Separate process per user+note (full isolation) | + +## Plugin System & Reflection Patterns + +### PluginManager — Custom Classloading + +`PluginManager` (`zeppelin-server/.../plugin/PluginManager.java`) loads plugins without Java SPI: + +``` +Plugin loading flow: +1. Check builtin list (hardcoded class names): + - Launchers: StandardInterpreterLauncher, SparkInterpreterLauncher + - NotebookRepos: VFSNotebookRepo, GitNotebookRepo + → if builtin: Class.forName(className) — direct classloading + +2. If not builtin → external plugin: + → Scan pluginsDir/{Launcher|NotebookRepo}/{pluginName}/ for JARs + → Create URLClassLoader with those JARs + → classLoader.loadClass(className) + → Instantiate via reflection (constructor parameters) +``` + +External plugin directory structure: +``` +plugins/ + Launcher/ + DockerInterpreterLauncher/ + *.jar + K8sStandardInterpreterLauncher/ + *.jar + NotebookRepo/ + S3NotebookRepo/ + *.jar + GCSNotebookRepo/ + *.jar +``` + +### ReflectionUtils + +`ReflectionUtils` (`zeppelin-server/.../util/ReflectionUtils.java`) provides generic reflection-based instantiation: + +```java +// No-arg constructor +ReflectionUtils.createClazzInstance(className) + +// Parameterized constructor +ReflectionUtils.createClazzInstance(className, parameterTypes, parameters) +``` + +Used to instantiate: +- `RecoveryStorage` — in `RemoteInterpreterServer` and `InterpreterSettingManager` +- `ConfigStorage` — in `InterpreterSettingManager` +- `LifecycleManager` — in `RemoteInterpreterServer` +- `NotebookRepo` — in `PluginManager` +- `InterpreterLauncher` — in `PluginManager` + +### Interpreter Discovery + +`InterpreterSettingManager` discovers interpreters at startup: + +``` +1. Scan interpreterDir (default: interpreter/) for subdirectories +2. For each subdirectory, look for interpreter-setting.json +3. Parse JSON → List +4. Register each interpreter's className, properties, editor settings +``` + +`interpreter-setting.json` format (in each interpreter module's resources): +```json +[{ + "group": "spark", + "name": "spark", + "className": "org.apache.zeppelin.spark.SparkInterpreter", + "properties": { + "spark.master": { "defaultValue": "local[*]", "description": "Spark master" } + }, + "editor": { "language": "scala", "editOnDblClick": false } +}] +``` + +### ZeppelinConfiguration Priority + +Configuration values are resolved in order (first match wins): +1. **Environment variables** (e.g., `ZEPPELIN_HOME`, `ZEPPELIN_PORT`) +2. **System properties** (e.g., `-Dzeppelin.server.port=8080`) +3. **zeppelin-site.xml** (`conf/zeppelin-site.xml`) +4. **Hardcoded defaults** (`ConfVars` enum in `ZeppelinConfiguration`) + +### HK2 Dependency Injection (zeppelin-server) + +`ZeppelinServer.startZeppelin()` sets up HK2 DI via `ServiceLocatorUtilities.bind()`: + +```java +new AbstractBinder() { + protected void configure() { + bind(storage).to(ConfigStorage.class); + bindAsContract(PluginManager.class).in(Singleton.class); + bindAsContract(InterpreterFactory.class).in(Singleton.class); + bindAsContract(NotebookRepoSync.class).to(NotebookRepo.class).in(Singleton.class); + bindAsContract(Notebook.class).in(Singleton.class); + // ... InterpreterSettingManager, SearchService, etc. + } +} +``` + +REST API classes use `@Inject` to receive these singletons. + +## Contributing Guide + +### Prerequisites + +| Tool | Version | Notes | +|------|---------|-------| +| JDK | pinned in `pom.xml` (`java.version`) | Required — use exactly that major, not a newer/older JDK | +| Maven | provided by `./mvnw` (pinned in `.mvn/wrapper/maven-wrapper.properties`) | No separate install needed | +| Node.js | see `zeppelin-web-angular/package.json` (`engines.node`) | Only for frontend (`zeppelin-web-angular/`) | + +### Initial Setup + +```bash +# Clone the repository +git clone https://github.com/apache/zeppelin.git +cd zeppelin + +# First build — skip tests to verify environment works +./mvnw clean package -DskipTests +# This takes ~10 minutes. If it succeeds, your environment is ready. + +# Frontend setup (only if working on UI) +cd zeppelin-web-angular +npm install +cd .. +``` + +### Development Workflow + +When starting a new change, use a **git worktree** instead of switching branches in your main checkout. This keeps your primary working directory clean and allows parallel work across multiple branches: + +```bash +# Create a worktree for your feature branch +git worktree add ../zeppelin-ZEPPELIN-XXXX -b ZEPPELIN-XXXX-description +cd ../zeppelin-ZEPPELIN-XXXX + +# When done, clean up +git worktree remove ../zeppelin-ZEPPELIN-XXXX +``` + +```bash +# Build only the module you're changing (--am builds required upstream modules) +./mvnw clean package -pl zeppelin-server --am -DskipTests + +# Run tests for your module +./mvnw test -pl zeppelin-server --am + +# Run a specific test +./mvnw test -pl zeppelin-server --am -Dtest=NotebookServerTest#testMethod + +# Start the dev frontend (proxies API to localhost:8080) +cd zeppelin-web-angular && npm start +``` + +For Spark or Flink work, add the version profile: +```bash +./mvnw clean package -pl spark -Pspark-3.5 -Pspark-scala-2.12 -DskipTests +``` + +### Before Submitting a PR + +1. **Write unit tests**. Every code change must include corresponding unit tests. Bug fixes should include a test that reproduces the bug. New features should have tests covering the main paths. + +2. **Run tests for affected modules**: + ```bash + ./mvnw test -pl + ``` + +3. **Check license headers** — all new files must have the Apache License 2.0 header: + ```bash + ./mvnw clean org.apache.rat:apache-rat-plugin:check -Prat + ``` + +4. **Lint frontend changes** (if applicable): + ```bash + cd zeppelin-web-angular && npm run lint:fix + ``` + +5. **Create a JIRA issue** at [issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN) and use the issue number in PR title: `[ZEPPELIN-XXXX] description`. + +### REST API Pattern + +All REST endpoints follow this pattern: + +```java +@Path("/notebook") +@Produces("application/json") +@Singleton +public class NotebookRestApi extends AbstractRestApi { + @Inject + public NotebookRestApi(Notebook notebook, ...) { + super(authenticationService); + } + + @GET + @Path("/{noteId}") + @ZeppelinApi + public Response getNote(@PathParam("noteId") String noteId) { + // Authorization check + checkIfUserCanRead(noteId, "Insufficient privileges"); + // Business logic via service layer + Note note = notebook.getNote(noteId); + // Return JsonResponse + return new JsonResponse<>(Status.OK, "", note).build(); + } +} +``` + +Key conventions: +- Extend `AbstractRestApi` (provides `getServiceContext()` for auth) +- Use `@Inject` constructor for HK2 DI +- Annotate public methods with `@ZeppelinApi` +- Return `JsonResponse(status, message, body).build()` +- Authorization via `checkIfUserCan{Read|Write|Run}()` + +### Code Style + +- **Java**: Google Java Style (2-space indent). Checkstyle enforced — no tabs, LF line endings, newline at EOF +- **Frontend**: ESLint + Prettier, auto-enforced via pre-commit hook (Husky + lint-staged) +- **Testing**: JUnit 5 (Jupiter) + Mockito (Java; a small number of legacy JUnit 4 tests still exist), Playwright (frontend E2E) +- **Logging**: SLF4J + Log4j2 +- **License**: Apache License 2.0 — all new files need the ASF header + +## Security + +Security model: [SECURITY.md](./SECURITY.md), which links to the project's +threat model at [THREAT_MODEL.md](./THREAT_MODEL.md). + +Agents that scan this repository should consult `THREAT_MODEL.md` for the +project's in-scope / out-of-scope declarations, the security properties it +provides and disclaims, the configuration knobs whose defaults change the +security envelope, and the known non-findings (recurring false positives) +before reporting issues. In particular, Apache Zeppelin executes user-supplied +notebook code through its interpreters by design — that is the product's +function, not a vulnerability; see `THREAT_MODEL.md` §3, §9, and §11a. diff --git a/Dockerfile b/Dockerfile index 982e54ed443..61a7714282c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM openjdk:11 as builder +FROM eclipse-temurin:11-jdk AS builder ADD . /workspace/zeppelin WORKDIR /workspace/zeppelin ENV MAVEN_OPTS="-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" diff --git a/LICENSE b/LICENSE index 3c3f246917d..c285c1b196a 100644 --- a/LICENSE +++ b/LICENSE @@ -277,3 +277,21 @@ Eclipse Public License - v 1.0 The following components are provided under the Eclipse Public License, version 1.0. See file headers and project links for details. (Eclipse Public License) pty4j - http://www.eclipse.org/legal/epl-v10.html + +======================================================================== +MIT License +======================================================================== +The following components are provided under the MIT License. See file headers and project links for details. + + (MIT License) ONNX Runtime (https://github.com/microsoft/onnxruntime) + Licensed under the MIT License. + https://github.com/microsoft/onnxruntime/blob/main/LICENSE + +======================================================================== +Apache License 2.0 (bundled dependencies) +======================================================================== +The following components are provided under the Apache License 2.0. See file headers and project links for details. + + (Apache License 2.0) DJL - Deep Java Library Tokenizers (https://github.com/deepjavalibrary/djl) + Licensed under the Apache License, Version 2.0. + https://github.com/deepjavalibrary/djl/blob/master/LICENSE diff --git a/NOTICE b/NOTICE index e583f549101..e1da12ea081 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Zeppelin -Copyright 2015 - 2025 The Apache Software Foundation +Copyright 2015 - 2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). @@ -12,3 +12,15 @@ Portions of this software were developed at NFLabs, Inc. (http://www.nflabs.com) * Pseudo terminal(PTY) implementation in Java * (Eclipse Public License) pty4j - http://www.eclipse.org/legal/epl-v10.html + +2. ONNX Runtime + + * Cross-platform ML inferencing and training accelerator + * (MIT License) onnxruntime - https://github.com/microsoft/onnxruntime + * Copyright (c) Microsoft Corporation + +3. Deep Java Library (DJL) HuggingFace Tokenizers + + * Java binding for HuggingFace tokenizers + * (Apache License 2.0) djl-tokenizers - https://github.com/deepjavalibrary/djl + * Copyright (c) Amazon.com, Inc. diff --git a/README.md b/README.md index 13a01d35294..7cb0bbc22fc 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,12 @@ **Zeppelin**, a web-based notebook that enables interactive data analytics. You can make beautiful data-driven, interactive and collaborative documents with SQL, Scala and more. Core features: - * Web based notebook style editor. - * Built-in Apache Spark support + * Web-based notebook style editor with real-time collaboration + * Multi-language support: Spark, Flink, Python, SQL, Shell, and 20+ interpreters + * Pluggable interpreter architecture with process isolation + * Built-in visualization and dynamic forms + * Notebook scheduling (cron) + * Flexible deployment: local, Docker, Kubernetes, YARN To know more about Zeppelin, visit our web site [https://zeppelin.apache.org](https://zeppelin.apache.org) diff --git a/Roadmap.md b/Roadmap.md index 95ec00f4ea1..0e08ccf54dc 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -1,4 +1,3 @@ - # Zeppelin Roadmap -Please check https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Roadmap for details \ No newline at end of file +Please check https://cwiki.apache.org/confluence/display/ZEPPELIN/Roadmap for details \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..b4d40854d0a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,44 @@ + + +# Security Policy + +## Reporting a Vulnerability + +Please report suspected security vulnerabilities in Apache Zeppelin privately +to the Apache Security Team at , following the ASF process +at . Do not open public GitHub issues or pull +requests for security reports. + +## Threat Model + +Apache Zeppelin's security threat model — what is in and out of scope, the +security properties the project provides and disclaims, the adversary model, +the configuration knobs whose defaults change the security envelope, and how +findings are triaged — is documented in [THREAT_MODEL.md](./THREAT_MODEL.md). + +Note that Apache Zeppelin executes user-supplied notebook code through its +interpreters by design; that capability is the product's primary function, and +the threat model is about *who may reach it and with what isolation*. See +`THREAT_MODEL.md` §3, §9, and §11a. + +## Operational Security Setup + +Operator-facing security configuration — authentication via Apache Shiro, +notebook authorization, interpreter user impersonation, and HTTPS / HTTP +security headers — is documented on the project website: + diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md new file mode 100644 index 00000000000..93a7389d1b9 --- /dev/null +++ b/THREAT_MODEL.md @@ -0,0 +1,410 @@ + + +# Apache Zeppelin — Security Threat Model + +## §1 Header + +- **Project:** Apache Zeppelin +- **Modeled against:** `master` HEAD as of 2026-06-05 (latest released docs line). +- **Authors:** ASF Security team (v0 draft, generated via the + `threat-model-producer` rubric), for the Apache Zeppelin PMC to review. +- **Status:** **v0 — PMC-reviewed.** The Apache Zeppelin PMC has reviewed the + framing (confirmed) and answered the §14 questions; the answers are recorded + inline in §14 and folded into the sections below, with confirmed claims + re-tagged *(maintainer)*. +- **Version binding:** this model is versioned with the project. A report + against Zeppelin release *N* is triaged against the model as it stood at *N*. +- **Reporting cross-reference:** findings that violate a §8 property should be + reported privately per the project's disclosure channel + (`security@apache.org`); findings that fall under §3 or §9 are closed citing + this document. +- **Provenance legend:** *(documented)* = stated in Zeppelin's own docs/site; + *(maintainer)* = confirmed by a Zeppelin PMC member; *(inferred)* = reasoned + from code/docs/domain knowledge, not yet confirmed (each has a §14 question). +- **Confidence:** ~18 documented; the §14 answers fold the bulk of the former + *(inferred)* claims to *(maintainer)*; a few loose environmental assumptions + remain *(inferred)*. + +**What Zeppelin is.** Apache Zeppelin is a web-based, multi-user notebook +server for interactive data analytics. Users open notebooks in a browser and +run "paragraphs" of code against pluggable *interpreters* (Spark, Flink, +Python, JDBC, shell, etc.); the Zeppelin server launches and talks to +interpreter processes over a Thrift IPC channel and returns results to the +browser over a websocket. **Executing user-supplied code on the server is the +product's primary function, not a vulnerability** — the security model is +about *who is allowed to reach that capability and with what isolation*, not +about preventing code execution. + +## §2 Scope and intended use + +- **Primary intended use** *(inferred)*: an operator-deployed, multi-tenant + analytics notebook server, run inside an organization's trusted network and + fronted by authentication, where authorized analysts author and run + notebooks against backend compute (Spark/Flink/etc.). +- **Deployment shape** *(documented)*: a long-running JVM server + (`zeppelin-server`) plus one or more interpreter processes; reached via HTTP + + websocket, optionally behind a reverse proxy (NGINX). +- **Caller roles** (a network service, so the role splits): + - **client / notebook user** — untrusted until authenticated; once + authenticated, trusted only up to their notebook/role permissions. + - **operator / admin** — trusted for the instance; owns `shiro.ini`, + interpreter settings, `zeppelin-site.xml`, the host. + - **anonymous visitor** — present **by default** (see §5a); trusted at + whatever level the deployment's authorization grants anonymous, which by + default is full access. + +**Component-family table** *(maintainer — §14.8)*: + +| Family | Entry point | Touches outside process? | In model? | +| --- | --- | --- | --- | +| Web/REST/websocket server | `org.apache.zeppelin.rest.*`, websocket | network | **yes** | +| AuthN/AuthZ (Shiro + notebook ACL + URL ACL) | `shiro.ini`, `NotebookAuthorization`, `SecurityRestApi` | filesystem (config) | **yes** | +| Interpreter launch + IPC | Thrift `RemoteInterpreterServer`, process launcher | child processes, network | **yes** (the launch/isolation boundary) | +| Interpreter-executed user code | `%spark`, `%sh`, `%python`, `%jdbc`, … | arbitrary (by design) | **boundary only** — the *code* is by-design; reaching/isolating it is in model | +| Credentials / datasource auth | `CredentialRestApi`, credential injection | filesystem, backends | **yes** | +| Notebook storage / repos | `NotebookRepo` (local FS, S3, Git, etc.) | filesystem / cloud | **yes** | +| Bundled interpreters / examples / web UI assets | `*-interpreter` modules, demos | varies | **all bundled interpreters first-class** for security purposes (§14.8); demo/example notebooks are a separate category | + +## §3 Out of scope (explicit non-goals) + +- **Sandboxing the code a permitted user runs.** A user with run permission on + a notebook can execute arbitrary code (`%sh`, Spark driver code, etc.) by + design; Zeppelin does not attempt to confine what that code does on the host + or backend. *(maintainer — §14.3)* +- **Defending a deployment that disables authentication and is exposed to an + untrusted network.** The docs direct operators to enable Shiro *or* deploy + only in a secured/trusted environment *(documented)*; an unauthenticated, + internet-exposed instance is an operator misconfiguration, not a Zeppelin + defect (pending the §5a/§14 ruling on whether anonymous is a supported + posture). +- **Security of third-party interpreter backends** (the Spark cluster, the + JDBC database, the host shell) — Zeppelin brokers access; it does not own + those systems' security. *(inferred)* +- **Bundled demo / example notebooks** — a separate category from the + interpreters, threat-modeled separately if at all; integrators should not + extend core guarantees to them. (All bundled *interpreters* are first-class; + see §2 and §14.8.) *(maintainer — §14.8)* + +## §4 Trust boundaries and data flow + +The **primary trust boundary is the authentication + authorization layer** +(Shiro realm → notebook ACL → URL ACL), not the API surface itself. Data flow +and the trust transitions it crosses: + +1. Browser → **HTTP/websocket** → server: crosses the network boundary. + Untrusted until Shiro authenticates the session. *(documented)* +2. Authenticated session → **notebook operation** (read/write/run): crosses + the notebook-ACL boundary (owner/reader/writer/runner). *(documented)* +3. Run request → **interpreter process** over Thrift IPC: the server hands + user code to an interpreter. Whether this crosses an OS-user boundary + depends on **impersonation** (off by default → runs as the *server* OS + user). *(documented)* +4. Interpreter → backend (Spark/JDBC/FS/shell): leaves Zeppelin's boundary + entirely. *(inferred)* + +**Reachability preconditions per family** (the triager's first test): +- A finding in the web/REST/websocket family is in-model only if reachable by + a network client **before** the Shiro auth gate, or by an authenticated user + **beyond** their granted role/notebook permission. +- A finding in interpreter-launch/IPC is in-model only if it lets a user cross + a boundary the model claims (e.g., one tenant reaching another tenant's + interpreter/credentials, or escaping the impersonation user when impersonation + is on). +- A finding that is "authenticated run-capable user executes code / reads files + as the interpreter's OS user" is **out of model** (that is the granted + capability) unless it crosses into another tenant or the operator boundary. + +## §5 Assumptions about the environment + +- **Operator-controlled host and config** *(inferred)*: `shiro.ini`, + `zeppelin-site.xml`, credential stores, and the interpreter settings are + trusted inputs written by the operator, not attacker-controllable. +- **Network placement** *(documented)*: Zeppelin expects to sit in a secured/ + trusted network or behind an authenticating proxy; HTTPS is "highly + recommended" for the web/websocket channel. +- **Backend trust** *(inferred)*: the Spark/Flink/JDBC/SSH backends an + interpreter reaches are provisioned by the operator. +- **What the server does to its host** *(inferred — §14)*: launches child + interpreter processes; reads config + notebook storage; opens listening + sockets (HTTP/websocket, Thrift); with impersonation, performs SSH/`setuid`- + style user switching. It is *not* expected to run as root. + +## §5a Build-time and configuration variants (the security-envelope knobs) + +This is the heart of Zeppelin's model — several **defaults are the less-secure +value**, so the model is ambiguous until the PMC rules on each (see §14 wave 1): + +| Knob | Default | Effect on model | Maintainer stance | +| --- | --- | --- | --- | +| Shiro authentication (`conf/shiro.ini`) | **absent → anonymous** *(documented)* | No auth boundary at all; every §8 authn/authz property is void | **dev-convenience** *(maintainer — §14.1)*: anonymous is *not* the supported posture; reports against an exposed anonymous instance are `OUT-OF-MODEL: non-default-build` | +| `zeppelin.notebook.public` / `ZEPPELIN_NOTEBOOK_PUBLIC` | **`true` → new notes public** *(documented)* | Empty-ACL note is readable/runnable by any authenticated (or anonymous) user | **by-design** *(maintainer — §14.2)*: public-by-default is intended; an empty-ACL note being readable/runnable is not a bug | +| Interpreter user impersonation | **off → runs as server OS user** *(documented)* | Without it, every run-capable user's code shares the *server's* OS identity/privileges and filesystem | **by-design** *(maintainer — §14.3)*: running as the server OS user is the documented default; OS isolation requires enabling impersonation | +| Interpreter binding mode (shared / scoped / isolated) | **shared** *(maintainer — §14.4)* | Process-level separation between users/notes; "isolated" is a *stability/resource* boundary, **not** a security sandbox | **maintainer — §14.4**: default is `shared`; no binding mode is a security sandbox | +| URL ACLs (`[urls]` in shiro.ini) gating `/interpreter`, `/credential`, `/configurations` | **not restricted unless operator adds them** *(documented)* | Sensitive admin endpoints open to any authenticated role absent explicit `[urls]` rules | **maintainer — §14.5**: no built-in admin gate; protection relies entirely on `shiro.ini [urls]` | +| HTTPS / security headers (`http_security_headers`) | **off/plain unless configured** *(documented)* | Credentials + session over plaintext; missing CSP/XFO | operator responsibility (§10); no CSP and Origin-based CSRF only — `VALID-HARDENING` *(maintainer — §14.10)* | + +**Insecure-default ruling (recorded).** The PMC has ruled that every insecure +§5a default above is a *dev-convenience / by-design* choice, not the supported +production posture: Zeppelin's stance is "open by default, secure by +configuration" (enable Shiro, or deploy only in a secured/trusted network). A +report that only manifests under one of these defaults is therefore +`OUT-OF-MODEL: non-default-build` (or `BY-DESIGN`), with the requirement living +in §10. See §14 wave 1 for the per-knob answers. + +## §6 Assumptions about inputs + +Inputs and their trust (network-service shape — rows are endpoints/messages): + +| Surface | Input | Attacker-controllable? | Caller/operator must enforce | +| --- | --- | --- | --- | +| `POST` login / Shiro filter | credentials | **yes** (pre-auth) | strong realm config; lockout/rate-limit at proxy *(inferred)* | +| Websocket ops (run/edit/move paragraph) | notebook + paragraph payload | **yes** (authenticated user) | notebook ACL + run permission enforced server-side *(documented)* | +| `NotebookRestApi` / `InterpreterRestApi` | note id, interpreter settings | **yes** (authenticated user) | URL ACL + ownership checks, enforced server-side *(maintainer — §14.6)* | +| `CredentialRestApi` | per-user credentials | **yes** (authenticated user) | per-user credential isolation *(maintainer — §14.7)* | +| Paragraph code body | arbitrary code | **yes — by design** | this is the granted capability, not validated input | +| `shiro.ini`, `zeppelin-site.xml`, interpreter JSON | config | **no — operator-trusted** | filesystem perms on config/secret files *(inferred)* | +| Notebook storage backend contents | persisted notes | **mostly trusted** (written via the app) | integrity of the repo (S3/Git/FS) *(inferred)* | + +Size/shape/rate: *(maintainer — §14.9)* there is no rate limit and no +concurrent-interpreter-launch cap today; the PMC treats this as +`VALID-HARDENING` and welcomes the scan surfacing concrete limits. + +## §7 Adversary model + +**In scope:** +- **Unauthenticated network client** (when Shiro is enabled): tries to reach + any authenticated capability without valid credentials — bypass the login + filter, forge/steal a session, reach a websocket/REST op pre-auth. +- **Authenticated lower-privileged user**: a legitimate user trying to exceed + their grant — read/edit/run another user's note they lack ACL for, read + another user's credentials, reach admin endpoints (`/interpreter`, + `/credential`) their role shouldn't, or (impersonation on) break out of their + impersonation identity into another user's or the server's. + +**Capabilities:** can send arbitrary HTTP/websocket traffic; can author +arbitrary code in paragraphs they may run; cannot (assumed) read the operator's +config/secret files or the host outside what their interpreter identity grants. + +**Explicitly out of scope:** +- The **operator/admin** and anyone with write access to `shiro.ini` / + interpreter config / the host — they have already won. +- A user **executing code within their own granted run permission** — that is + the product working as designed, even though the code can be `%sh rm -rf`. +- Anyone reaching a **default anonymous** instance over an untrusted network — + out of model pending the §5a/§14 ruling. + +## §8 Security properties the project provides + +Each conditional on the relevant §5a knob being set securely. The PMC has +confirmed properties 1–5 below as committed properties (§14.5–§14.7), now +tagged *(maintainer)*; property 6 (resource/availability) is **not** a committed +property today (§14.9). + +1. **Authentication of the web/REST/websocket surface** *when Shiro is + configured*. Violation symptom: an unauthenticated client performs an + operation requiring a session. Severity: **critical**. *(maintainer — + §14.6)* +2. **Authorization of notebook operations per the owner/reader/writer/runner + ACL** *when auth is on*. Violation symptom: a user reads/edits/runs a note + they lack permission for. Severity: **critical**. *(maintainer — §14.6: + enforced server-side for every websocket/REST op, not client-side only)* +3. **URL-level access control** for sensitive endpoints via `[urls]`. + Violation symptom: a non-admin reaches `/interpreter`, `/credential`, or + `/configurations` despite a restricting rule. Severity: **high**. + *(documented mechanism; maintainer — §14.5: no built-in admin gate, so this + property holds only when the operator adds `[urls]` rules)* +4. **Per-user credential isolation** (one user cannot read another's injected + datasource credentials). Violation symptom: cross-user credential read. + Severity: **critical**. *(maintainer — §14.7)* +5. **Impersonation confinement** *when enabled*: interpreter code runs as the + logged-in user, not the server user, and not as another user. Violation + symptom: code runs as a different identity than the session's. Severity: + **high**. *(maintainer — §14.3)* +6. **Resource/availability** — **not a committed property today** *(maintainer — + §14.9)*. There is no rate limit or concurrent-launch cap. The PMC treats + hardening here as `VALID-HARDENING` and welcomes concrete recommendations + from the scan rather than suppressing them. + +## §9 Security properties the project does *not* provide + +- **No sandbox of permitted code.** A run-capable user's paragraph executes + with the full authority of the interpreter's OS identity (the *server* user + unless impersonation is on). "Zeppelin runs my shell command" is not a + vulnerability. *(inferred — §14.3)* +- **No protection in anonymous/default mode.** With no `shiro.ini`, there is no + authn/authz boundary; everything in §8 is void. *(documented)* +- **No transport security by default.** Plaintext HTTP/websocket unless the + operator configures TLS; credentials transit in the clear otherwise. + *(documented)* + +**False-friend properties (call out explicitly):** +- **Interpreter "isolated" binding mode is not a security sandbox.** It gives + each user/note a separate interpreter *process* for stability and resource + separation; it does **not** confine what the code in that process can do to + the host or to shared backends, and absent impersonation all those processes + still run as the **same server OS user**. *(maintainer — §14.4)* +- **Notebook permissions are an application-layer ACL, not OS isolation.** A + user denied *read* on a note in the UI may still reach data through an + interpreter they *can* run if backends aren't separately access-controlled. + *(maintainer)* + +**Well-known attack classes left to the operator/integrator:** SSRF from +interpreter code reaching internal services; secrets-in-notebooks; XSS/CSRF on +the notebook web UI — *(maintainer — §14.10)* there is **no Content-Security- +Policy** and CSRF protection is **Origin-header-based only**, so strengthening +these (CSP, stronger CSRF) is welcome `VALID-HARDENING`; websocket cross-origin. +The point is to put integrators on notice. + +## §10 Downstream / operator responsibilities + +For Zeppelin the "user" is the **operator** deploying it: +- **Enable Shiro authentication** (or keep Zeppelin strictly inside a trusted, + network-isolated perimeter). *(documented)* +- **Add `[urls]` rules** restricting `/interpreter`, `/credential`, + `/configurations` (and other admin paths) to admin roles. *(documented)* +- **Set `zeppelin.notebook.public=false`** if notebooks should default to + private. *(documented)* +- **Enable interpreter impersonation** for genuine multi-tenant isolation; do + not rely on binding mode alone. *(documented)* +- **Terminate TLS** and enable `http_security_headers`. *(documented)* +- **Protect `shiro.ini`, credential stores, and notebook storage** with host + filesystem permissions; do not run the server as root. *(inferred)* +- **Treat backends (Spark/JDBC/SSH) as reachable by any run-capable user** and + access-control them independently. *(inferred)* + +## §11 Known misuse patterns + +- Exposing a **default (anonymous) Zeppelin to the public internet** — turns + the by-design code-execution surface into unauthenticated RCE-equivalent. +- Relying on **notebook ACLs while interpreters run shared/as-server-user**, so + a run-capable user reaches data the ACL meant to hide. +- Treating **"isolated" binding as a security boundary** between tenants. +- Storing **long-lived secrets in notebook source** instead of the credential + store, then sharing the note. +- Leaving **`/interpreter` and `/credential` reachable** by all authenticated + users (no `[urls]` rules). + +## §11a Known non-findings (recurring false positives) + +The highest-leverage section for keeping scan output signal-heavy: + +- **"`%sh` / interpreter executes arbitrary shell or driver code → RCE."** + By design for a run-capable user; `OUT-OF-MODEL` / `BY-DESIGN` unless it + crosses a tenant or the operator boundary. (§3, §9) *(maintainer — §14.3)* +- **"Interpreter process runs as the Zeppelin server OS user / can read server + files."** Documented default behavior without impersonation; operator config, + not a defect. (§5a, §10) *(documented; maintainer — §14.3)* +- **"Anonymous user can do X"** reported against a deployment with **no + `shiro.ini`.** Out of model — auth is operator-enabled. (§5a, §9) + *(maintainer — §14.1)* +- **"No TLS / credentials in plaintext"** against a deployment the operator did + not configure for HTTPS. Operator responsibility. (§10) *(documented)* +- **Static-analysis "command injection / code execution" hits on the + interpreter execution path.** That path *is* the feature; in-model only if it + bypasses the authn/authz gate. (§4 reachability test) *(maintainer)* + +## §12 Conditions that would change this model + +- A new network surface or REST/websocket endpoint; a new first-class + interpreter; a change of any §5a default (e.g., shipping `shiro.ini` enabled, + or `notebook.public=false` by default); adding a built-in sandbox for + interpreter code; a new notebook-storage backend with different trust. +- **A report that cannot be routed to one §13 disposition** is itself evidence + the model is incomplete — revise §8/§9 rather than make an ad-hoc call. + +## §13 Triage dispositions + +| Disposition | Meaning | Licensed by | +| --- | --- | --- | +| `VALID` | Bypasses the authn/authz gate, or lets a user exceed their notebook/role grant, or crosses a tenant/credential/impersonation boundary, with auth configured. | §8, §6, §7 | +| `VALID-HARDENING` | No §8 property broken, but the API makes a §11 misuse too easy; hardened at maintainer discretion. | §11 | +| `OUT-OF-MODEL: trusted-input` | Requires control of operator config (`shiro.ini`, interpreter JSON, host). | §6 | +| `OUT-OF-MODEL: adversary-not-in-scope` | Requires operator/admin privilege, or is a run-capable user executing code within their grant. | §7 | +| `OUT-OF-MODEL: non-default-build` | Only manifests under an insecure §5a default the PMC rules dev-only (e.g., anonymous mode, public notebooks, no impersonation). | §5a | +| `BY-DESIGN: property-disclaimed` | Concerns interpreter code execution or a property §9 disclaims. | §9 | +| `KNOWN-NON-FINDING` | Matches a §11a pattern. | §11a | +| `MODEL-GAP` | Cannot be routed above → revise the model. | §12 | + +## §14 Open questions for the maintainers + +Grouped in waves; each states the **proposed answer** followed by the **PMC +answer** (recorded by the Apache Zeppelin PMC, 2026-06-11). The core framing — +RBAC (Shiro + notebook ACL + URL ACL + impersonation) is the trust boundary, +not a sandbox, and a `%sh` from a run-capable user is the product working, not +RCE — was confirmed by the PMC and is kept. + +**Wave 1 — scope & the insecure defaults (these reshape everything):** +1. **Anonymous default.** Proposed: anonymous/no-`shiro.ini` is a *dev- + convenience*; the supported production posture requires Shiro **or** a + trusted isolated network. So reports against an internet-exposed anonymous + instance are `OUT-OF-MODEL: non-default-build`. Correct? (→ §5a, §3, §11a) + **→ PMC: confirmed.** +2. **`notebook.public=true` default.** Proposed: public-by-default is intended + convenience; operators needing isolation set it false. A "any user can read + an empty-ACL note" report is by-design, not a bug. Correct? (→ §5a, §2) + **→ PMC: confirmed — by-design.** +3. **Impersonation off by default.** Proposed: without impersonation, all + interpreter code legitimately runs as the **server** OS user; this is the + documented default and not a vulnerability; multi-tenant OS isolation + requires enabling impersonation. Correct? (→ §3, §5a, §9, §11a) + **→ PMC: confirmed.** +4. **Binding mode as boundary.** Proposed: shared/scoped/isolated are + stability/resource controls, **not** security sandboxes; we should state + that explicitly in §9. Agree? Which is the default? (→ §5a, §9) + **→ PMC: agreed — the default is `shared`, and no binding mode is a security + sandbox.** + +**Wave 2 — properties & enforcement:** +5. **URL ACL default.** Are `/interpreter`, `/credential`, `/configurations` + open to any authenticated role unless `[urls]` restricts them, or is there a + built-in admin gate? (→ §5a, §8) + **→ PMC: no built-in admin gate — it relies on `shiro.ini [urls]`.** +6. **Server-side ACL enforcement.** Are notebook ACLs + role checks enforced on + the **server** for every websocket/REST op (not just hidden in the UI)? Any + ops that check only client-side? (→ §6, §8) + **→ PMC: enforced server-side; there are no client-side-only checks.** +7. **Credential isolation.** Does the credential store guarantee one user + cannot read another user's injected credentials, including via a shared + interpreter process? (→ §8, §9) + **→ PMC: yes — per-user credentials are isolated.** + +**Wave 3 — surfaces & limits:** +8. **First-class interpreters.** Which interpreters/modules are supported for + security purposes vs. community/unsupported (→ §2/§3 carve-out)? + **→ PMC: all bundled interpreters are supported for security purposes + (first-class).** +9. **Resource limits.** Any limits on paragraph/result size, websocket rate, or + concurrent interpreter launches? Where's the line between in-model pre-auth + exhaustion and by-design expensive queries? (→ §6, §8) + **→ PMC: `VALID-HARDENING` — please surface. No rate limit or + concurrent-launch cap today; this is an area we would like the scan to flag + and recommend improvements for.** +10. **Web-UI hardening.** Does enabling `http_security_headers` give CSRF + XSS + + clickjacking coverage, or are those partly the operator's job? (→ §9) + **→ PMC: `VALID-HARDENING` — please surface. No CSP, and CSRF is + Origin-based only; concrete improvements from the scan are welcome.** +11. **Coexistence.** This is a new `THREAT_MODEL.md`; `SECURITY.md` (currently a + stub) should point at it as canonical, and the website security pages stay + the operator how-to. Agree? (→ meta) + **→ PMC: agreed.** + +## §15 Machine-readable companion + +Deferred for v0; a `threat-model.yaml` sidecar (entry points → trust, §5a +defaults, §8 properties, §11a suppressions, §13 labels) can be generated once +the prose is ratified. diff --git a/alluxio/pom.xml b/alluxio/pom.xml deleted file mode 100644 index 5edc2118a39..00000000000 --- a/alluxio/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - 4.0.0 - - - zeppelin-interpreter-parent - org.apache.zeppelin - 0.13.0-SNAPSHOT - ../zeppelin-interpreter-parent/pom.xml - - - zeppelin-alluxio - jar - Zeppelin: Alluxio interpreter - - - 2.9.0 - alluxio - - - - - org.alluxio - alluxio-shell - ${alluxio.version} - - - - - com.google.guava - guava - 31.0.1-jre - test - - - com.google.protobuf - protobuf-java - 3.16.1 - test - - - - org.mockito - mockito-core - test - - - - org.alluxio - alluxio-minicluster - ${alluxio.version} - test - - - org.apache.hadoop - hadoop-client - - - - - - org.apache.hadoop - hadoop-client-api - test - - - - org.apache.hadoop - hadoop-client-runtime - test - - - - - - - maven-enforcer-plugin - - - maven-resources-plugin - - - maven-shade-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - - diff --git a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java deleted file mode 100644 index 5478926ec9d..00000000000 --- a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.alluxio; - -import alluxio.cli.fs.FileSystemShell; -import alluxio.conf.Configuration; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; -import java.util.stream.Stream; - - -import org.apache.zeppelin.completer.CompletionType; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; - -/** - * Alluxio interpreter for Zeppelin. - */ -public class AlluxioInterpreter extends Interpreter { - - private static final Logger LOGGER = LoggerFactory.getLogger(AlluxioInterpreter.class); - - protected static final String ALLUXIO_MASTER_HOSTNAME = "alluxio.master.hostname"; - protected static final String ALLUXIO_MASTER_PORT = "alluxio.master.port"; - - private FileSystemShell fs; - - private int totalCommands = 0; - private int completedCommands = 0; - - private final String alluxioMasterHostname; - private final String alluxioMasterPort; - - protected final List keywords = Arrays.asList("cat", "chgrp", - "chmod", "chown", "copyFromLocal", "copyToLocal", "count", - "createLineage", "deleteLineage", "du", "fileInfo", "free", - "getCapacityBytes", "getUsedBytes", "listLineages", "load", - "loadMetadata", "location", "ls", "mkdir", "mount", "mv", - "persist", "pin", "report", "rm", "setTtl", "tail", "touch", - "unmount", "unpin", "unsetTtl"); - - public AlluxioInterpreter(Properties property) { - super(property); - - alluxioMasterHostname = property.getProperty(ALLUXIO_MASTER_HOSTNAME); - alluxioMasterPort = property.getProperty(ALLUXIO_MASTER_PORT); - } - - private Stream filteredProperties(String prefix) { - return properties.stringPropertyNames().stream().filter( - propertyKey -> propertyKey.startsWith(prefix) - ); - } - - @Override - public void open() { - LOGGER.info("Starting Alluxio shell to connect to " + alluxioMasterHostname + - " on port " + alluxioMasterPort); - // Setting the extra parameters being set in the interpreter config starting with alluxio - filteredProperties("alluxio.").forEach(x -> System.setProperty(x, properties.getProperty(x))); - - System.setProperty(PropertyKey.USER_RPC_RETRY_MAX_DURATION.getName(), "5s"); - - AlluxioConfiguration conf = Configuration.global(); - // Reduce the RPC retry max duration to fall earlier for CLIs - fs = new FileSystemShell(conf); - } - - @Override - public void close() { - LOGGER.info("Closing Alluxio shell"); - try { - fs.close(); - } catch (IOException e) { - LOGGER.error("Cannot close connection", e); - } - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - String[] lines = splitAndRemoveEmpty(st, "\n"); - return interpret(lines, context); - } - - private InterpreterResult interpret(String[] commands, InterpreterContext context) { - boolean isSuccess = true; - totalCommands = commands.length; - completedCommands = 0; - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos); - PrintStream old = System.out; - - System.setOut(ps); - - for (String command : commands) { - int commandResult = 1; - String[] args = splitAndRemoveEmpty(command, " "); - if (args.length > 0 && args[0].equals("help")) { - System.out.println(getCommandList()); - } else { - commandResult = fs.run(args); - } - if (commandResult != 0) { - isSuccess = false; - break; - } else { - completedCommands += 1; - } - System.out.println(); - } - - System.out.flush(); - System.setOut(old); - - if (isSuccess) { - return new InterpreterResult(Code.SUCCESS, baos.toString()); - } else { - return new InterpreterResult(Code.ERROR, baos.toString()); - } - } - - private String[] splitAndRemoveEmpty(String st, String splitSeparator) { - String[] voices = st.split(splitSeparator); - ArrayList result = new ArrayList<>(); - for (String voice : voices) { - if (!voice.trim().isEmpty()) { - result.add(voice); - } - } - return result.toArray(new String[result.size()]); - } - - private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) { - ArrayList result = new ArrayList<>(); - for (String st : sts) { - result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator))); - } - return result.toArray(new String[result.size()]); - } - - @Override - public void cancel(InterpreterContext context) { } - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - @Override - public int getProgress(InterpreterContext context) { - return completedCommands * 100 / totalCommands; - } - - @Override - public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { - String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " "); - String lastWord = ""; - if (words.length > 0) { - lastWord = words[ words.length - 1 ]; - } - - List voices = new LinkedList<>(); - for (String command : keywords) { - if (command.startsWith(lastWord)) { - voices.add(new InterpreterCompletion(command, command, CompletionType.command.name())); - } - } - return voices; - } - - private String getCommandList() { - StringBuilder sb = new StringBuilder(); - sb.append("Commands list:"); - sb.append("\n\t[help] - List all available commands."); - sb.append("\n\t[cat ] - Prints the file's contents to the console."); - sb.append("\n\t[chgrp [-R] ] - Changes the group of a file or directory " + - "specified by args. Specify -R to change the group recursively."); - sb.append("\n\t[chmod -R ] - Changes the permission of a file or directory " + - "specified by args. Specify -R to change the permission recursively."); - sb.append("\n\t[chown -R ] - Changes the owner of a file or directory " + - "specified by args. Specify -R to change the owner recursively."); - sb.append("\n\t[copyFromLocal ] - Copies a file or a directory from " + - "local filesystem to Alluxio filesystem."); - sb.append("\n\t[copyToLocal ] - Copies a file or a directory from the " + - "Alluxio filesystem to the local filesystem."); - sb.append("\n\t[count ] - Displays the number of files and directories matching " + - "the specified prefix."); - sb.append("\n\t[createLineage " + - "[ ...]] - Creates a lineage."); - sb.append("\n\t[deleteLineage ] - Deletes a lineage. If " + - "cascade is specified as true, dependent lineages will also be deleted."); - sb.append("\n\t[du ] - Displays the size of the specified file or directory."); - sb.append("\n\t[fileInfo ] - Displays all block info for the specified file."); - sb.append("\n\t[free ] - Removes the file or directory(recursively) " + - "from Alluxio memory space."); - sb.append("\n\t[getCapacityBytes] - Gets the capacity of the Alluxio file system."); - sb.append("\n\t[getUsedBytes] - Gets number of bytes used in the Alluxio file system."); - sb.append("\n\t[listLineages] - Lists all lineages."); - sb.append("\n\t[load ] - Loads a file or directory in Alluxio space, makes it " + - "resident in memory."); - sb.append("\n\t[loadMetadata ] - Loads metadata for the given Alluxio path from the " + - "under file system."); - sb.append("\n\t[location ] - Displays the list of hosts storing the specified file."); - sb.append("\n\t[ls [-R] ] - Displays information for all files and directories " + - "directly under the specified path. Specify -R to display files and " + - "directories recursively."); - sb.append("\n\t[mkdir [path2] ... [pathn]] - Creates the specified directories, " + - "including any parent directories that are required."); - sb.append("\n\t[mount ] - Mounts a UFS path onto an Alluxio path."); - sb.append("\n\t[mv ] - Renames a file or directory."); - sb.append("\n\t[persist ] - Persists a file or directory currently stored " + - "only in Alluxio to the UnderFileSystem."); - sb.append("\n\t[pin ] - Pins the given file or directory in memory (works " + - "recursively for directories). Pinned files are never evicted from memory, unless " + - "TTL is set."); - sb.append("\n\t[report ] - Reports to the master that a file is lost."); - sb.append("\n\t[rm [-R] ] - Removes the specified file. Specify -R to remove file or " + - "directory recursively."); - sb.append("\n\t[setTtl

Instead of hard-coding your CQL queries, it is possible to use - Zeppelin dynamic form + Zeppelin dynamic form syntax to inject simple value or multiple choices forms. The legacy mustache syntax ( {{ }} ) to bind input text and select form is still supported but is deprecated and will be removed in future releases. @@ -1050,7 +1050,7 @@ select id, double, float, text, date, time, timestamp from zep.test_format;Asynchronous execution is only possible when it is possible to return a Future value in the InterpreterResult. It may be an interesting proposal for the Zeppelin project.

Recently, Zeppelin allows you to choose the level of isolation for your interpreters (see - Interpreter Binding Mode ). + Interpreter Binding Mode ).

Long story short, you have 3 available bindings: diff --git a/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java b/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java index 8a6cce4ee9e..4426d47d00e 100644 --- a/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java +++ b/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java @@ -27,13 +27,13 @@ import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.cassandraunit.CQLDataLoader; -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.cassandraunit.utils.EmbeddedCassandraServerHelper; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.testcontainers.containers.CassandraContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -65,28 +65,40 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -public class CassandraInterpreterTest { // extends AbstractCassandraUnit4CQLTestCase { +@Testcontainers +public class CassandraInterpreterTest { private static final String ARTISTS_TABLE = "zeppelin.artists"; private static volatile CassandraInterpreter interpreter; + private static CqlSession session; + private final InterpreterContext intrContext = InterpreterContext.builder() .setParagraphTitle("Paragraph1") .build(); + @Container + public static CassandraContainer cassandra = + new CassandraContainer<>("cassandra:4.1.3"); + @BeforeAll - public static synchronized void setUp() throws IOException, InterruptedException { - System.setProperty("cassandra.skip_wait_for_gossip_to_settle", "0"); - System.setProperty("cassandra.load_ring_state", "false"); - System.setProperty("cassandra.initial_token", "0"); - System.setProperty("cassandra.num_tokens", "nil"); - System.setProperty("cassandra.allocate_tokens_for_local_replication_factor", "nil"); - EmbeddedCassandraServerHelper.startEmbeddedCassandra(); - CqlSession session = EmbeddedCassandraServerHelper.getSession(); - new CQLDataLoader(session).load(new ClassPathCQLDataSet("prepare_all.cql", "zeppelin")); + public static synchronized void setUp() throws IOException { + session = CqlSession.builder() + .addContactPoint(java.net.InetSocketAddress.createUnresolved( + cassandra.getHost(), cassandra.getMappedPort(9042))) + .withLocalDatacenter("datacenter1") + .build(); + + String cql = IOUtils.resourceToString("/prepare_all.cql", StandardCharsets.UTF_8); + for (String stmt : cql.split(";")) { + String trimmed = stmt.trim(); + if (!trimmed.isEmpty()) { + session.execute(trimmed); + } + } Properties properties = new Properties(); - properties.setProperty(CASSANDRA_CLUSTER_NAME, EmbeddedCassandraServerHelper.getClusterName()); + properties.setProperty(CASSANDRA_CLUSTER_NAME, "Test Cluster"); properties.setProperty(CASSANDRA_COMPRESSION_PROTOCOL, "NONE"); properties.setProperty(CASSANDRA_CREDENTIALS_USERNAME, "none"); properties.setProperty(CASSANDRA_CREDENTIALS_PASSWORD, "none"); @@ -111,9 +123,9 @@ public static synchronized void setUp() throws IOException, InterruptedException properties.setProperty(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS, "12000"); properties.setProperty(CASSANDRA_SOCKET_TCP_NO_DELAY, "true"); - properties.setProperty(CASSANDRA_HOSTS, EmbeddedCassandraServerHelper.getHost()); + properties.setProperty(CASSANDRA_HOSTS, cassandra.getHost()); properties.setProperty(CASSANDRA_PORT, - Integer.toString(EmbeddedCassandraServerHelper.getNativeTransportPort())); + Integer.toString(cassandra.getMappedPort(9042))); properties.setProperty("datastax-java-driver.advanced.connection.pool.local.size", "1"); interpreter = new CassandraInterpreter(properties); interpreter.open(); @@ -122,6 +134,9 @@ public static synchronized void setUp() throws IOException, InterruptedException @AfterAll public static void tearDown() { interpreter.close(); + if (session != null) { + session.close(); + } } @Test @@ -333,7 +348,7 @@ void should_execute_statement_with_timestamp_option() throws Exception { String statement2 = "@timestamp=15\n" + "INSERT INTO zeppelin.ts(key,val) VALUES('k','v2');"; - CqlSession session = EmbeddedCassandraServerHelper.getSession(); + CqlSession session = CassandraInterpreterTest.session; // Insert v1 with current timestamp interpreter.interpret(statement1, intrContext); System.out.println("going to read data from zeppelin.ts;"); @@ -562,14 +577,17 @@ void should_display_statistics_for_non_select_statement() { // When final InterpreterResult actual = interpreter.interpret(query, intrContext); - final int port = EmbeddedCassandraServerHelper.getNativeTransportPort(); - final String address = EmbeddedCassandraServerHelper.getHost(); + final int port = cassandra.getMappedPort(9042); + final String address = cassandra.getHost(); // Then final String expected = rawResult.replaceAll("TRIED_HOSTS", address + ":" + port) .replaceAll("QUERIED_HOSTS", address + ":" + port); assertEquals(Code.SUCCESS, actual.code()); - assertEquals(expected, reformatHtml(actual.message().get(0).getData())); + // JDK 17+ renders unresolved InetSocketAddress as "host/:port" + String actualHtml = reformatHtml(actual.message().get(0).getData()) + .replaceAll(address + "/<unresolved>:", address + ":"); + assertEquals(expected, actualHtml); } @Test diff --git a/cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html b/cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html index 8d721ef2338..ed67b250cd3 100644 --- a/cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html +++ b/cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html @@ -1 +1 @@ -


  live_data

ReplicationDurable Writes
{'class' : 'org.apache.cassandra.locator.SimpleStrategy', 'replication_factor' : '1'}false

Tables
Column TypeColumn NameData Type
pk1uuid
pk2int
my_static1text
my_static2text
 clustering1timestamp
 clustering2int
 clustering3text
entries_indexed_mapmap<int, text>
indexed1text
indexed2int
key_indexed_mapmap<int, text>
my_listlist<text>
my_mapmap<int, text>
my_udtfrozen<live_data.address>
my_udt_listfrozen<list<frozen<live_data.address>>>
simpledouble

 complex_table's indices

NameTarget
clustering2idxclustering2
idx1indexed1
idx2indexed2
keys_map_idxkeys(key_indexed_map)
pk2idxpk2
Column TypeColumn NameData Type
sensor_iduuid
monthint
characteristicsmap<text, text>
model_numbertext
providertext
 datetimestamp
valuedouble
Column TypeColumn NameData Type
station_iduuid
sensorsfrozen<map<uuid, frozen<live_data.geolocation>>>

User Defined Types
Column NameData Type
numberint
streettext
zipint
citytext
countrytext
Column NameData Type
latitudedouble
longitudedouble
\ No newline at end of file +


  live_data

ReplicationDurable Writes
{'class' : 'org.apache.cassandra.locator.SimpleStrategy', 'replication_factor' : '1'}false

Tables
Column TypeColumn NameData Type
pk1uuid
pk2int
my_static1text
my_static2text
 clustering1timestamp
 clustering2int
 clustering3text
entries_indexed_mapmap<int, text>
indexed1text
indexed2int
key_indexed_mapmap<int, text>
my_listlist<text>
my_mapmap<int, text>
my_udtfrozen<live_data.address>
my_udt_listfrozen<list<frozen<live_data.address>>>
simpledouble

 complex_table's indices

NameTarget
clustering2idxclustering2
idx1indexed1
idx2indexed2
keys_map_idxkeys(key_indexed_map)
pk2idxpk2
Column TypeColumn NameData Type
sensor_iduuid
monthint
characteristicsmap<text, text>
model_numbertext
providertext
 datetimestamp
valuedouble
Column TypeColumn NameData Type
station_iduuid
sensorsfrozen<map<uuid, frozen<live_data.geolocation>>>

User Defined Types
Column NameData Type
numberint
streettext
zipint
citytext
countrytext
Column NameData Type
latitudedouble
longitudedouble
\ No newline at end of file diff --git a/cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html b/cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html index b31dc11ee55..09cb963f474 100644 --- a/cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html +++ b/cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html @@ -1 +1 @@ -


 complex_table

Column TypeColumn NameData Type
pk1uuid
pk2int
my_static1text
my_static2text
 clustering1timestamp
 clustering2int
 clustering3text
entries_indexed_mapmap<int, text>
indexed1text
indexed2int
key_indexed_mapmap<int, text>
my_listlist<text>
my_mapmap<int, text>
my_udtfrozen<live_data.address>
my_udt_listfrozen<list<frozen<live_data.address>>>
simpledouble

 complex_table's indices

NameTarget
clustering2idxclustering2
idx1indexed1
idx2indexed2
keys_map_idxkeys(key_indexed_map)
pk2idxpk2
\ No newline at end of file +


 complex_table

Column TypeColumn NameData Type
pk1uuid
pk2int
my_static1text
my_static2text
 clustering1timestamp
 clustering2int
 clustering3text
entries_indexed_mapmap<int, text>
indexed1text
indexed2int
key_indexed_mapmap<int, text>
my_listlist<text>
my_mapmap<int, text>
my_udtfrozen<live_data.address>
my_udt_listfrozen<list<frozen<live_data.address>>>
simpledouble

 complex_table's indices

NameTarget
clustering2idxclustering2
idx1indexed1
idx2indexed2
keys_map_idxkeys(key_indexed_map)
pk2idxpk2
\ No newline at end of file diff --git a/cassandra/src/test/resources/scalate/NoResultWithExecutionInfo.html b/cassandra/src/test/resources/scalate/NoResultWithExecutionInfo.html index bd713adad2e..f15b05dcaee 100644 --- a/cassandra/src/test/resources/scalate/NoResultWithExecutionInfo.html +++ b/cassandra/src/test/resources/scalate/NoResultWithExecutionInfo.html @@ -1 +1 @@ -
No Result      
InformationValue
StatementCREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);
Tried Hostslocalhost:9142
Queried Hostslocalhost:9142
Schema in Agreementtrue
\ No newline at end of file +
No Result      
InformationValue
StatementCREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);
Tried HostsTRIED_HOSTS
Queried HostsQUERIED_HOSTS
Schema in Agreementtrue
\ No newline at end of file diff --git a/conf/interpreter-list b/conf/interpreter-list index 792b6d6858f..c6e0305f602 100644 --- a/conf/interpreter-list +++ b/conf/interpreter-list @@ -17,7 +17,6 @@ # # [name] [maven artifact] [description] -alluxio org.apache.zeppelin:zeppelin-alluxio:0.13.0-SNAPSHOT Alluxio interpreter angular org.apache.zeppelin:zeppelin-angular:0.13.0-SNAPSHOT HTML and AngularJS view rendering bigquery org.apache.zeppelin:zeppelin-bigquery:0.13.0-SNAPSHOT BigQuery interpreter cassandra org.apache.zeppelin:zeppelin-cassandra:0.13.0-SNAPSHOT Cassandra interpreter diff --git a/conf/shiro.ini.template b/conf/shiro.ini.template index 6721d175f91..c60f278d148 100644 --- a/conf/shiro.ini.template +++ b/conf/shiro.ini.template @@ -30,7 +30,7 @@ user3 = password4, role2 #activeDirectoryRealm = org.apache.zeppelin.realm.ActiveDirectoryGroupRealm #activeDirectoryRealm.systemUsername = userNameA -#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html +#use either systemPassword or hadoopSecurityCredentialPath, more details in https://zeppelin.apache.org/docs/latest/setup/security/shiro_authentication.html #activeDirectoryRealm.systemPassword = passwordA #activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks #activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM @@ -87,6 +87,9 @@ sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager cookie = org.apache.shiro.web.servlet.SimpleCookie cookie.name = JSESSIONID cookie.httpOnly = true +### Restrict the session cookie to same-site requests by default. Set to NONE only when +### Zeppelin is intentionally embedded into a different origin (and 'cookie.secure = true'). +cookie.sameSite = LAX ### Uncomment the below line only when Zeppelin is running over HTTPS #cookie.secure = true sessionManager.sessionIdCookie = $cookie diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index d5e54b91f16..d04aee833e6 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -576,7 +576,9 @@ zeppelin.interpreter.lifecyclemanager.class org.apache.zeppelin.interpreter.lifecycle.TimeoutLifecycleManager LifecycleManager class for managing the lifecycle of interpreters, by default interpreter will - be closed after timeout + be closed after timeout. With TimeoutLifecycleManager, Zeppelin server tracks the last use of each + interpreter group and closes the idle ones itself, so the threshold below can be overridden per + interpreter setting @@ -588,7 +590,9 @@ zeppelin.interpreter.lifecyclemanager.timeout.threshold 1h - Interpreter timeout threshold, by default it is 1 hour + Interpreter timeout threshold, by default it is 1 hour. Set the same property on an + individual interpreter setting to override it for that interpreter only, or set it to 0 there to + keep that interpreter from ever being reclaimed --> diff --git a/dev/create_release.sh b/dev/create_release.sh index b5529169f6c..0622497b9ee 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -97,7 +97,7 @@ function make_binary_release() { git_clone make_source_package -make_binary_release netinst "-Pweb-classic -pl !hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!alluxio,!livy,!groovy,!java,!neo4j,!sparql,!mongodb,!shell -am" +make_binary_release netinst "-Pweb-classic -pl !hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!livy,!groovy,!java,!neo4j,!sparql,!mongodb,!shell -am" make_binary_release all "-Pweb-classic -pl !shell" diff --git a/dev/merge_pr.py b/dev/merge_pr.py new file mode 100644 index 00000000000..4fbfac77f6d --- /dev/null +++ b/dev/merge_pr.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""merge_pr.py - Merge Apache Zeppelin pull requests via the GitHub API. + +Optionally cherry-picks into release branches and resolves JIRA issues. +No external dependencies — uses only Python 3 built-in libraries. + +Usage: + python3 dev/merge_pr.py --pr 5167 --dry-run + python3 dev/merge_pr.py --pr 5167 --resolve-jira --fix-versions 0.13.0 + python3 dev/merge_pr.py --pr 5167 --resolve-jira --release-branches branch-0.12 +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.request + +GITHUB_API_BASE = "https://api.github.com/repos/apache/zeppelin" +JIRA_API_BASE = "https://issues.apache.org/jira/rest/api/2" + +DEFAULT_BRANCH = "master" +DEFAULT_REMOTE = "apache" +JIRA_RESOLVE_TRANSITION = "Resolve Issue" +JIRA_CLOSED_STATUSES = frozenset(("Resolved", "Closed")) + +JIRA_ID_RE = re.compile(r"ZEPPELIN-\d{3,6}") +TITLE_FORMATTED_RE = re.compile(r"^\[ZEPPELIN-\d{3,6}](\[[A-Z0-9_\s,]+] )+\S+") +TITLE_REF_RE = re.compile(r"(?i)(ZEPPELIN[-\s]*\d{3,6})") +COMPONENT_RE = re.compile(r"(?i)(\[[\w\s,.\-]+])") +WHITESPACE_RE = re.compile(r"\s+") +LEADING_NON_WORD_RE = re.compile(r"^\W+") +SEMANTIC_VER_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +class MergePR: + def __init__(self, args): + self.pr = args.pr + self.target = args.target or "" + self.fix_versions = _parse_csv(args.fix_versions) if args.fix_versions else [] + self.release_branches = _parse_csv(args.release_branches) if args.release_branches else [] + self.resolve_jira = args.resolve_jira + self.dry_run = args.dry_run + self.push_remote = args.push_remote or os.environ.get("PUSH_REMOTE_NAME", DEFAULT_REMOTE) + self.github_token = args.github_token or os.environ.get("GITHUB_OAUTH_KEY", "") + self.jira_token = args.jira_token or os.environ.get("JIRA_ACCESS_TOKEN", "") + + # ── Git ────────────────────────────────────────────────────────────── + + def _git(self, *args): + result = subprocess.run( + ["git", *args], + capture_output=True, text=True, + ) + if result.returncode != 0: + output = (result.stdout + result.stderr).strip() + raise RuntimeError(f"git {' '.join(args)} failed:\n{output}") + return result.stdout.strip() + + def _git_current_ref(self): + ref = self._git("rev-parse", "--abbrev-ref", "HEAD") + return self._git("rev-parse", "HEAD") if ref == "HEAD" else ref + + # ── HTTP ───────────────────────────────────────────────────────────── + + def _http(self, method, url, payload=None, auth=""): + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + if auth: + req.add_header("Authorization", auth) + try: + with urllib.request.urlopen(req) as resp: + body = resp.read().decode() + return resp.status, json.loads(body) if body else {} + except urllib.error.HTTPError as e: + err_body = e.read().decode() if e.fp else "" + try: + return e.code, json.loads(err_body) if err_body else {} + except json.JSONDecodeError: + return e.code, {"error": err_body} + + # ── GitHub ─────────────────────────────────────────────────────────── + + def _gh_auth(self): + return f"token {self.github_token}" if self.github_token else "" + + def _gh_get_pr(self, num): + code, data = self._http("GET", f"{GITHUB_API_BASE}/pulls/{num}", auth=self._gh_auth()) + if code != 200: + raise RuntimeError(f"GET PR #{num}: HTTP {code}") + return data + + def _gh_merge_pr(self, num, title, msg): + payload = {"commit_title": title, "commit_message": msg, "merge_method": "squash"} + code, data = self._http("PUT", f"{GITHUB_API_BASE}/pulls/{num}/merge", payload, self._gh_auth()) + if code == 405: + raise RuntimeError(f"Merge PR #{num} is not allowed") + if code != 200: + raise RuntimeError(f"Merge PR #{num}: HTTP {code}") + return data + + def _gh_comment_pr(self, num, comment): + code, _ = self._http("POST", f"{GITHUB_API_BASE}/issues/{num}/comments", + {"body": comment}, self._gh_auth()) + if code != 201: + print(f"Warning: comment PR #{num}: HTTP {code}", file=sys.stderr) + + # ── JIRA ───────────────────────────────────────────────────────────── + + def _jira_auth(self): + return f"Bearer {self.jira_token}" if self.jira_token else "" + + def _jira_get_issue(self, key): + code, data = self._http("GET", f"{JIRA_API_BASE}/issue/{key}", auth=self._jira_auth()) + if code != 200: + raise RuntimeError(f"GET {key}: HTTP {code}") + return data + + def _jira_unreleased_versions(self): + code, data = self._http("GET", f"{JIRA_API_BASE}/project/ZEPPELIN/versions", auth=self._jira_auth()) + if code != 200: + raise RuntimeError(f"GET versions: HTTP {code}") + versions = [] + for v in data: + name = v.get("name", "") + if not v.get("released") and not v.get("archived") and SEMANTIC_VER_RE.match(name): + versions.append({"id": str(v["id"]), "name": name}) + versions.sort(key=lambda v: _ver_tuple(v["name"]), reverse=True) + return versions + + def _jira_transitions(self, key): + code, data = self._http("GET", f"{JIRA_API_BASE}/issue/{key}/transitions", auth=self._jira_auth()) + if code != 200: + raise RuntimeError(f"GET transitions {key}: HTTP {code}") + return [{"id": t["id"], "name": t["name"]} for t in data.get("transitions", [])] + + def _jira_resolve(self, key, transition_id, fix_ver, comment): + payload = { + "transition": {"id": transition_id}, + "update": { + "comment": [{"add": {"body": comment}}], + "fixVersions": [{"add": {"id": fv["id"], "name": fv["name"]}} for fv in fix_ver], + }, + } + code, _ = self._http("POST", f"{JIRA_API_BASE}/issue/{key}/transitions", payload, self._jira_auth()) + if code != 204: + raise RuntimeError(f"Resolve {key}: HTTP {code}") + + # ── Fix version resolution ─────────────────────────────────────────── + + def _resolve_fix_versions(self, branches, versions): + """Resolve fix version objects from explicit --fix-versions and branch inference. + + Returns a list of version dicts ({"id": ..., "name": ...}). + Raises RuntimeError if an explicit fix version is not found. + """ + vm = {v["name"]: v for v in versions} + fix_ver, seen = [], set() + + for fv in self.fix_versions: + if fv not in vm: + raise RuntimeError(f'fix version "{fv}" not found') + fix_ver.append(vm[fv]) + seen.add(fv) + + infer_master = not self.fix_versions + latest = versions[0]["name"] + names = [] + for branch in branches: + if branch == DEFAULT_BRANCH: + if infer_master and latest not in seen: + names.append(latest) + seen.add(latest) + else: + prefix = branch[len("branch-"):] if branch.startswith("branch-") else branch + found = [v["name"] for v in versions if v["name"].startswith(prefix + ".") or v["name"] == prefix] + if found: + pick = found[-1] # smallest matching (list is desc-sorted) + if pick not in seen: + names.append(pick) + seen.add(pick) + else: + print(f"Warning: no version found for {branch}, skipping", file=sys.stderr) + + # Remove redundant X.Y.0 when X.(Y-1).0 is also present + filtered = [] + for v in names: + parts = v.split(".") + if len(parts) == 3 and parts[2] == "0": + minor = int(parts[1]) + if minor > 0 and f"{parts[0]}.{minor - 1}.0" in seen: + continue + filtered.append(v) + + inferred = [vm[n] for n in filtered if n in vm] + if inferred: + print(f"Auto-inferred fix version(s): {', '.join(filtered)}") + fix_ver.extend(inferred) + return fix_ver + + # ── Effective command ──────────────────────────────────────────────── + + def _print_effective_command(self, target_branch, fix_ver): + parts = ["python3 dev/merge_pr.py", f"--pr {self.pr}"] + if target_branch and target_branch != DEFAULT_BRANCH: + parts.append(f"--target {target_branch}") + if self.release_branches: + parts.append(f"--release-branches {','.join(self.release_branches)}") + if self.resolve_jira: + parts.append("--resolve-jira") + if fix_ver: + parts.append(f"--fix-versions {','.join(fv['name'] for fv in fix_ver)}") + if self.push_remote != DEFAULT_REMOTE: + parts.append(f"--push-remote {self.push_remote}") + print(f"[dry-run] Effective command:\n {' '.join(parts)}") + + # ── Main flow ──────────────────────────────────────────────────────── + + def run(self): + original_head = self._git_current_ref() + + pr_data = self._gh_get_pr(self.pr) + if not pr_data.get("mergeable"): + raise RuntimeError(f"PR #{self.pr} is not mergeable") + pr_title = pr_data["title"] + if "[WIP]" in pr_title: + print(f"WARNING: PR title contains [WIP]: {pr_title}", file=sys.stderr) + + target_branch = self.target or pr_data["base"]["ref"] + title = _standardize_title(pr_title) + src = f"{pr_data['user']['login']}/{pr_data['head']['ref']}" + pr_body = pr_data.get("body", "") or "" + + print(f"=== Pull Request #{self.pr} ===") + print(f"title: {title}") + print(f"source: {src}") + print(f"target: {target_branch}") + print(f"url: {pr_data['url']}") + if self.release_branches: + print(f"release-branches: {', '.join(self.release_branches)}") + + # Resolve fix versions once (used for both dry-run display and actual JIRA resolution) + fix_ver = [] + if self.resolve_jira and self.jira_token and JIRA_ID_RE.search(title): + try: + versions = self._jira_unreleased_versions() + if versions: + branches = [target_branch] + self.release_branches + fix_ver = self._resolve_fix_versions(branches, versions) + except RuntimeError as e: + print(f"Warning: failed to resolve fix versions: {e}", file=sys.stderr) + + if self.dry_run: + print() + self._print_effective_command(target_branch, fix_ver) + return + + # Merge + body = pr_body.replace("@", "") + try: + name = self._git("config", "--get", "user.name") + except RuntimeError: + name = "" + try: + email = self._git("config", "--get", "user.email") + except RuntimeError: + email = "" + msg = f"{body}\n\nCloses #{self.pr} from {src}.\n\nSigned-off-by: {name} <{email}>" + + merge_data = self._gh_merge_pr(self.pr, title, msg) + sha = merge_data["sha"] + print(f"\nPR #{self.pr} merged! (hash: {_short_sha(sha)})") + + try: + self._git("fetch", self.push_remote, target_branch) + except RuntimeError: + pass + + # Cherry-pick into release branches + merged = [target_branch] + for branch in self.release_branches: + pick = _pick_branch_name(self.pr, branch) + try: + self._git("fetch", self.push_remote, f"{branch}:{pick}") + except RuntimeError as e: + print(f"Warning: fetch {branch} failed: {e}", file=sys.stderr) + continue + self._git("checkout", pick) + try: + self._git("cherry-pick", "-sx", sha) + self._git("push", self.push_remote, f"{pick}:{branch}") + h = self._git("rev-parse", pick) + print(f"Picked into {branch} (hash: {_short_sha(h)})") + merged.append(branch) + except RuntimeError as e: + print(f"Warning: cherry-pick/push into {branch} failed: {e}", file=sys.stderr) + try: + self._git("cherry-pick", "--abort") + except RuntimeError: + pass + finally: + self._git("checkout", original_head) + self._git("branch", "-D", pick) + + self._comment_merge_summary(merged, sha) + + if self.resolve_jira: + try: + self._do_resolve_jira(title, fix_ver) + except RuntimeError as e: + print(f"Warning: JIRA resolution failed: {e}", file=sys.stderr) + + def _comment_merge_summary(self, merged, sha): + lines = [f"Merged into {merged[0]} ({_short_sha(sha)})."] + for branch in merged[1:]: + lines.append(f"Cherry-picked into {branch}.") + try: + self._gh_comment_pr(self.pr, "\n".join(lines)) + print("Commented on PR with merge summary.") + except RuntimeError as e: + print(f"Warning: failed to comment on PR: {e}", file=sys.stderr) + + def _do_resolve_jira(self, title, fix_ver): + if not self.jira_token: + raise RuntimeError("JIRA_ACCESS_TOKEN is not set") + + ids = JIRA_ID_RE.findall(title) + if not ids: + print("No JIRA ID found in PR title, skipping.") + return + + for jira_id in ids: + try: + issue = self._jira_get_issue(jira_id) + except RuntimeError as e: + print(f"Warning: get {jira_id}: {e}", file=sys.stderr) + continue + status = issue.get("fields", {}).get("status", {}).get("name", "") + if status in JIRA_CLOSED_STATUSES: + print(f'JIRA {jira_id} already "{status}", skipping.') + continue + + print(f"=== JIRA {jira_id} ===") + print(f"Summary: {issue.get('fields', {}).get('summary', '')}") + print(f"Status: {status}") + + transitions = self._jira_transitions(jira_id) + resolve_id = next((t["id"] for t in transitions if t["name"] == JIRA_RESOLVE_TRANSITION), None) + if not resolve_id: + print(f"Warning: no '{JIRA_RESOLVE_TRANSITION}' transition for {jira_id}", file=sys.stderr) + continue + + jira_comment = ( + f"Issue resolved by pull request {self.pr}" + f"\n[https://github.com/apache/zeppelin/pull/{self.pr}]" + ) + try: + self._jira_resolve(jira_id, resolve_id, fix_ver, jira_comment) + print(f"Resolved {jira_id}!") + except RuntimeError as e: + print(f"Warning: resolve {jira_id}: {e}", file=sys.stderr) + + +# ── Module-level utilities ─────────────────────────────────────────────── + +def _parse_csv(value): + return [s.strip() for s in value.split(",") if s.strip()] if value else [] + + +def _ver_tuple(v): + return tuple(int(x) for x in v.split(".")) + + +def _short_sha(sha): + return sha[:8] if len(sha) > 8 else sha + + +def _pick_branch_name(pr_num, branch): + return f"PR_TOOL_PICK_PR_{pr_num}_{branch.upper()}" + + +def _standardize_title(text): + text = text.rstrip(".") + if text.startswith('Revert "') and text.endswith('"'): + return text + if TITLE_FORMATTED_RE.match(text): + return text + + jira_refs = [] + for m in TITLE_REF_RE.finditer(text): + ref = m.group(1) + jira_refs.append("[" + WHITESPACE_RE.sub("-", ref.upper()) + "]") + text = text.replace(ref, "") + + components = [] + for m in COMPONENT_RE.finditer(text): + comp = m.group(1) + components.append(comp.upper()) + text = text.replace(comp, "") + + text = LEADING_NON_WORD_RE.sub("", text) + result = "".join(jira_refs) + "".join(components) + " " + text + return WHITESPACE_RE.sub(" ", result.strip()) + + +# ── Entry point ────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Merge Apache Zeppelin pull requests", + usage="python3 dev/merge_pr.py [flags]", + ) + parser.add_argument("--pr", type=int, required=True, help="Pull request number") + parser.add_argument("--target", default="", help="Target branch (default: PR base branch)") + parser.add_argument("--fix-versions", default="", help="JIRA fix version(s), comma-separated") + parser.add_argument("--release-branches", default="", help="Release branch(es) to cherry-pick into, comma-separated") + parser.add_argument("--resolve-jira", action="store_true", help="Resolve associated JIRA issue(s)") + parser.add_argument("--dry-run", action="store_true", help="Show what would be done without making changes") + parser.add_argument("--push-remote", default="", help="Git remote for pushing (default: apache)") + parser.add_argument("--github-token", default="", help="GitHub OAuth token (env: GITHUB_OAUTH_KEY)") + parser.add_argument("--jira-token", default="", help="JIRA access token (env: JIRA_ACCESS_TOKEN)") + + args = parser.parse_args() + MergePR(args).run() + + +if __name__ == "__main__": + main() diff --git a/dev/requirements.txt b/dev/requirements.txt index c1aa4ef539d..4fef9cef477 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -21,9 +21,9 @@ idna==3.10 jira==3.10.5 oauthlib==3.3.1 packaging==25.0 -python-dotenv==1.1.1 -requests==2.32.5 +python-dotenv==1.2.2 +requests==2.33.0 requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 typing-extensions==4.14.1 -urllib3==2.5.0 +urllib3==2.6.3 diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 00000000000..7f2b05cef5e --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,198 @@ + + +# AGENTS.md + +> Scoped guidance for work under `docs/`. This file complements the +> repository-root `AGENTS.md`. + +## Scope And Ownership + +- `docs/` is the source for Apache Zeppelin's versioned product documentation. +- The main `zeppelin.apache.org` website is maintained in + `apache/zeppelin-site`; its homepage does not need to use the same generator + as these versioned docs. +- Markdown, layouts, includes, and assets in this directory are built here. + The generated site is written to `docs/_site/`. +- `docs/_site/` is generated and gitignored. Never edit or commit it. + +## Build Model + +The current build is: + +```text +docs sources + docs/_config.yml + -> Jekyll from docs/Gemfile.lock + -> docs/_site/ + -> zeppelin-site/docs// during a separate publication step +``` + +- `Gemfile` declares Jekyll and its documentation build dependencies. +- `Gemfile.lock` pins the actual Ruby dependency versions. The Docker commands + use `bundle exec` so the pinned Jekyll version is used. +- `_config.yml` supplies `ZEPPELIN_VERSION` and `JB.BASE_PATH`. +- `_includes/JB/setup` applies `JB.BASE_PATH` only for a safe build. Therefore + a publication build must include `--safe`. +- `Rakefile` contains legacy Jekyll-Bootstrap helpers. It is not the primary + build entry point; use the Docker commands below. +- The Maven build does not generate this site. +- Docker is the supported build environment. Do not install or run Ruby, + Bundler, or Jekyll directly on the host. + +## Preview And Build + +Preview with Docker: + +```bash +cd docs +docker run --rm -it \ + --user "$(id -u):$(id -g)" \ + -e HOME=/usr/local/bundle \ + -e BUNDLE_FROZEN=true \ + -v "$PWD:/docs" \ + -w /docs \ + -p '4000:4000' \ + ruby:4.0.6 \ + bash -lc "bundle install && bundle exec jekyll serve --watch --host 0.0.0.0" +``` + +Open `http://localhost:4000`. The preview intentionally runs without +`--safe`, so links are rooted at `/` instead of the production version path. +The container uses the current user's UID and GID so generated files remain +owned by that user on the host. The Ruby image's writable gem directory is +also used as the container home for that user. + +Build the publication artifact with Docker: + +```bash +cd docs +docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e HOME=/usr/local/bundle \ + -e BUNDLE_FROZEN=true \ + -v "$PWD:/docs" \ + -w /docs \ + ruby:4.0.6 \ + bash -lc "bundle install && bundle exec jekyll build --safe" +``` + +The output must be under `_site/`, and generated links and assets must use the +`JB.BASE_PATH` configured in `_config.yml`. + +When `Gemfile` changes, update `Gemfile.lock` inside Docker: + +```bash +cd docs +docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e HOME=/usr/local/bundle \ + -v "$PWD:/docs" \ + -w /docs \ + ruby:4.0.6 \ + bundle lock --update +``` + +Run the publication build after updating the lockfile. + +## Authoring Conventions + +- Preserve the ASF license header in every new source file. +- Follow the front matter used by nearby pages: + + ```yaml + --- + layout: page + title: "Page title" + description: "Short description" + group: section/subsection + --- + ``` + +- Include `{% include JB/setup %}` before page content when following the + existing page layout. +- Prefix internal site links and assets with `{{BASE_PATH}}` when an absolute + site path is needed. Production docs are hosted below `/docs//`, + not at the domain root. +- Update `_includes/themes/zeppelin/_navigation.html` when a page must appear + in the global documentation navigation. +- Keep filenames, headings, and link targets stable unless the task explicitly + includes redirects or link migration. +- Check the corresponding source code or configuration template when + documenting runtime behavior. Do not infer current behavior from an older + documentation page. + +## Version Handling + +- `ZEPPELIN_VERSION` and `JB.BASE_PATH` in `_config.yml` must identify the same + version. +- `dev/change_zeppelin_version.sh` updates both values as part of a repository + version change. Do not change them for an ordinary documentation edit. +- Before producing release docs, verify that `JB.BASE_PATH` is exactly + `/docs/`. + +## Publication Boundary + +- Building this directory does not publish the website. +- The generated `_site/` tree is copied into + `apache/zeppelin-site/docs//` by separate release/site work. +- The `zeppelin-site` repository owns the homepage, ASF staging/publishing, + and the mapping or redirect for `/docs/latest/`. +- Do not modify `zeppelin-site`, historical documentation snapshots, or + publication branches unless the user explicitly includes that work. + +## ASF Website Policy + +- Follow the ASF project website policy at + `https://privacy.apache.org/policies/website-policy.html` and the Infra CSP + guidance at `https://infra.apache.org/csp.html`. +- Do not add Google Analytics or any other third-party analytics, tracker, + tracking pixel, advertising tag, or external monitoring script. +- Do not load JavaScript, CSS, fonts, images, or other assets from non-ASF + domains. Host an asset in this repository when its license permits, or use a + normal external link instead of embedding it. +- Third-party embeds require the consent and DPA handling described by the ASF + policy. Prefer a direct link unless the task explicitly includes an approved + consent flow. +- The production layout uses the ASF-hosted Matomo instance provisioned for + Apache Zeppelin as site ID `69`. Do not replace it with another analytics + service or change its endpoint without Privacy team approval. + +## Verification + +For every documentation change: + +1. Run the Docker publication build above from `docs/`. +2. Confirm `_site/index.html` and the generated file for each changed page + exist. +3. Check generated navigation, links, images, and code blocks for the affected + pages. +4. Confirm generated URLs use the configured `/docs//` prefix. +5. Check the generated site for external trackers and embedded resources: + + ```bash + docker run --rm \ + -v "$PWD:/docs:ro" \ + -w /docs \ + ruby:4.0.6 \ + ruby check_external_resources.rb _site + ``` + +6. Run `git status --short` and keep `_site/` and incidental dependency changes + out of the commit. + +For navigation, layout, CSS, or JavaScript changes, also run the preview server +and inspect the affected pages at desktop and narrow viewport widths. diff --git a/docs/Gemfile b/docs/Gemfile index 620e29f3b0f..160ec9c8fef 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -14,10 +14,8 @@ # limitations under the License. # source 'https://rubygems.org' -gem 'github-pages' -gem 'redcarpet' -gem 'jekyll-twitter-plugin' -gem 'nokogiri', '1.15.4' +gem 'jekyll', '4.4.1' +gem 'nokogiri', '1.19.3' gem 'mini_portile2', '2.8.4' gem "webrick", "~> 1.8" # By upgrading ruby to 3.x diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 9475b4a3ee8..7ac28335bcb 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,272 +1,98 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.2.1) - base64 - bigdecimal - concurrent-ruby (~> 1.0, >= 1.3.1) - connection_pool (>= 2.2.5) - drb - i18n (>= 1.6, < 2) - logger (>= 1.4.2) - minitest (>= 5.1) - securerandom (>= 0.3) - tzinfo (~> 2.0, >= 2.0.5) - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - base64 (0.2.0) - bigdecimal (3.1.8) - coffee-script (2.4.1) - coffee-script-source - execjs - coffee-script-source (1.12.2) + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + bigdecimal (4.1.2) colorator (1.1.0) - commonmarker (0.23.10) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) - dnsruby (1.72.2) - simpleidn (~> 0.2.1) - drb (2.2.1) + concurrent-ruby (1.3.8) + csv (3.3.5) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) - ethon (0.16.0) - ffi (>= 1.15.0) eventmachine (1.2.7) - execjs (2.9.1) - faraday (2.12.0) - faraday-net_http (>= 2.0, < 3.4) - json - logger - faraday-net_http (3.3.0) - net-http - ffi (1.17.0) - ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.4) + ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86_64-linux-gnu) forwardable-extended (2.6.0) - gemoji (4.1.0) - github-pages (231) - github-pages-health-check (= 1.18.2) - jekyll (= 3.9.5) - jekyll-avatar (= 0.8.0) - jekyll-coffeescript (= 1.2.2) - jekyll-commonmark-ghpages (= 0.4.0) - jekyll-default-layout (= 0.1.5) - jekyll-feed (= 0.17.0) - jekyll-gist (= 1.5.0) - jekyll-github-metadata (= 2.16.1) - jekyll-include-cache (= 0.2.1) - jekyll-mentions (= 1.6.0) - jekyll-optional-front-matter (= 0.3.2) - jekyll-paginate (= 1.1.0) - jekyll-readme-index (= 0.3.0) - jekyll-redirect-from (= 0.16.0) - jekyll-relative-links (= 0.6.1) - jekyll-remote-theme (= 0.4.3) - jekyll-sass-converter (= 1.5.2) - jekyll-seo-tag (= 2.8.0) - jekyll-sitemap (= 1.4.0) - jekyll-swiss (= 1.0.0) - jekyll-theme-architect (= 0.2.0) - jekyll-theme-cayman (= 0.2.0) - jekyll-theme-dinky (= 0.2.0) - jekyll-theme-hacker (= 0.2.0) - jekyll-theme-leap-day (= 0.2.0) - jekyll-theme-merlot (= 0.2.0) - jekyll-theme-midnight (= 0.2.0) - jekyll-theme-minimal (= 0.2.0) - jekyll-theme-modernist (= 0.2.0) - jekyll-theme-primer (= 0.6.0) - jekyll-theme-slate (= 0.2.0) - jekyll-theme-tactile (= 0.2.0) - jekyll-theme-time-machine (= 0.2.0) - jekyll-titles-from-headings (= 0.5.3) - jemoji (= 0.13.0) - kramdown (= 2.4.0) - kramdown-parser-gfm (= 1.1.0) - liquid (= 4.0.4) - mercenary (~> 0.3) - minima (= 2.5.1) - nokogiri (>= 1.13.6, < 2.0) - rouge (= 3.30.0) - terminal-table (~> 1.4) - github-pages-health-check (1.18.2) - addressable (~> 2.3) - dnsruby (~> 1.60) - octokit (>= 4, < 8) - public_suffix (>= 3.0, < 6.0) - typhoeus (~> 1.3) - html-pipeline (2.14.3) - activesupport (>= 2) - nokogiri (>= 1.4) - http_parser.rb (0.8.0) - i18n (1.14.6) + google-protobuf (4.35.1) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-arm64-darwin) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86_64-linux-gnu) + bigdecimal + rake (~> 13.3) + http_parser.rb (0.8.1) + i18n (1.15.2) concurrent-ruby (~> 1.0) - jekyll (3.9.5) + jekyll (4.4.1) addressable (~> 2.4) + base64 (~> 0.2) colorator (~> 1.0) + csv (~> 3.0) em-websocket (~> 0.5) - i18n (>= 0.7, < 2) - jekyll-sass-converter (~> 1.0) + i18n (~> 1.0) + jekyll-sass-converter (>= 2.0, < 4.0) jekyll-watch (~> 2.0) - kramdown (>= 1.17, < 3) + json (~> 2.6) + kramdown (~> 2.3, >= 2.3.1) + kramdown-parser-gfm (~> 1.0) liquid (~> 4.0) - mercenary (~> 0.3.3) + mercenary (~> 0.3, >= 0.3.6) pathutil (~> 0.9) - rouge (>= 1.7, < 4) + rouge (>= 3.0, < 5.0) safe_yaml (~> 1.0) - jekyll-avatar (0.8.0) - jekyll (>= 3.0, < 5.0) - jekyll-coffeescript (1.2.2) - coffee-script (~> 2.2) - coffee-script-source (~> 1.12) - jekyll-commonmark (1.4.0) - commonmarker (~> 0.22) - jekyll-commonmark-ghpages (0.4.0) - commonmarker (~> 0.23.7) - jekyll (~> 3.9.0) - jekyll-commonmark (~> 1.4.0) - rouge (>= 2.0, < 5.0) - jekyll-default-layout (0.1.5) - jekyll (>= 3.0, < 5.0) - jekyll-feed (0.17.0) - jekyll (>= 3.7, < 5.0) - jekyll-gist (1.5.0) - octokit (~> 4.2) - jekyll-github-metadata (2.16.1) - jekyll (>= 3.4, < 5.0) - octokit (>= 4, < 7, != 4.4.0) - jekyll-include-cache (0.2.1) - jekyll (>= 3.7, < 5.0) - jekyll-mentions (1.6.0) - html-pipeline (~> 2.3) - jekyll (>= 3.7, < 5.0) - jekyll-optional-front-matter (0.3.2) - jekyll (>= 3.0, < 5.0) - jekyll-paginate (1.1.0) - jekyll-readme-index (0.3.0) - jekyll (>= 3.0, < 5.0) - jekyll-redirect-from (0.16.0) - jekyll (>= 3.3, < 5.0) - jekyll-relative-links (0.6.1) - jekyll (>= 3.3, < 5.0) - jekyll-remote-theme (0.4.3) - addressable (~> 2.0) - jekyll (>= 3.5, < 5.0) - jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) - rubyzip (>= 1.3.0, < 3.0) - jekyll-sass-converter (1.5.2) - sass (~> 3.4) - jekyll-seo-tag (2.8.0) - jekyll (>= 3.8, < 5.0) - jekyll-sitemap (1.4.0) - jekyll (>= 3.7, < 5.0) - jekyll-swiss (1.0.0) - jekyll-theme-architect (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.6.0) - jekyll (> 3.5, < 5.0) - jekyll-github-metadata (~> 2.9) - jekyll-seo-tag (~> 2.0) - jekyll-theme-slate (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-titles-from-headings (0.5.3) - jekyll (>= 3.3, < 5.0) - jekyll-twitter-plugin (2.1.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-sass-converter (3.1.0) + sass-embedded (~> 1.75) jekyll-watch (2.2.1) listen (~> 3.0) - jemoji (0.13.0) - gemoji (>= 3, < 5) - html-pipeline (~> 2.2) - jekyll (>= 3.0, < 5.0) - json (2.7.2) - kramdown (2.4.0) - rexml + json (2.21.1) + kramdown (2.5.2) + rexml (>= 3.4.4) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) liquid (4.0.4) - listen (3.9.0) + listen (3.10.0) + logger rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - logger (1.6.1) - mercenary (0.3.6) + logger (1.7.0) + mercenary (0.4.0) mini_portile2 (2.8.4) - minima (2.5.1) - jekyll (>= 3.5, < 5.0) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) - minitest (5.25.1) - net-http (0.4.1) - uri - nokogiri (1.15.4) + nokogiri (1.19.3) mini_portile2 (~> 2.8.2) racc (~> 1.4) - octokit (4.25.1) - faraday (>= 1, < 3) - sawyer (~> 0.9) + nokogiri (1.19.3-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.3-x86_64-linux-gnu) + racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (5.1.1) + public_suffix (7.0.5) racc (1.8.1) + rake (13.4.2) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - redcarpet (3.6.0) - rexml (3.3.8) - rouge (3.30.0) - rubyzip (2.3.2) + rexml (3.4.4) + rouge (4.7.0) safe_yaml (1.0.5) - sass (3.7.4) - sass-listen (~> 4.0.0) - sass-listen (4.0.0) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.9.2) - addressable (>= 2.3.5) - faraday (>= 0.17.3, < 3) - securerandom (0.3.1) - simpleidn (0.2.3) - terminal-table (1.8.0) - unicode-display_width (~> 1.1, >= 1.1.1) - typhoeus (1.4.1) - ethon (>= 0.9.0) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (1.8.0) - uri (0.13.1) - webrick (1.8.2) + sass-embedded (1.102.0) + google-protobuf (~> 4.31) + rake (>= 13) + sass-embedded (1.102.0-arm64-darwin) + google-protobuf (~> 4.31) + sass-embedded (1.102.0-x86_64-linux-gnu) + google-protobuf (~> 4.31) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.6.0) + webrick (1.9.2) PLATFORMS arm64-darwin @@ -274,12 +100,10 @@ PLATFORMS x86_64-linux DEPENDENCIES - github-pages - jekyll-twitter-plugin + jekyll (= 4.4.1) mini_portile2 (= 2.8.4) - nokogiri (= 1.15.4) - redcarpet + nokogiri (= 1.19.3) webrick (~> 1.8) BUNDLED WITH - 2.5.16 + 4.0.16 diff --git a/docs/README.md b/docs/README.md index 1f67945f04d..a14a176e442 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,79 +3,68 @@ This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/](https://zeppelin.apache.org/docs/latest/) is also generated from the files found in here. ## Build documentation -Zeppelin is using [Jekyll](https://jekyllrb.com/) which is a static site generator and [Github Pages](https://pages.github.com/) as a site publisher. For the more details, see [help.github.com/articles/about-github-pages-and-jekyll/](https://help.github.com/articles/about-github-pages-and-jekyll/). +Zeppelin uses [Jekyll](https://jekyllrb.com/) to generate the static versioned documentation published on the Apache Zeppelin website. **Requirements** -``` -# ruby --version >= 2.0.0 -# Install Bundler using gem -gem install bundler - -cd $ZEPPELIN_HOME/docs -# Install all dependencies declared in the Gemfile -bundle install -``` - -For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements). - -On OS X 10.9, you may need to do +- [Docker](https://docs.docker.com/get-docker/) -``` -xcode-select --install -``` - -**Docker** +Ruby, Bundler, and Jekyll run only inside the Docker container. No host Ruby +installation is required. -Local docker environments are also supported and have been tested using: -* [Docker version 20.10.2](https://docs.docker.com/get-docker/) +## Preview documentation -## Run website locally -If you don't want to encounter ugly rendered pages, run the documentation site in your local environment first. +From `$ZEPPELIN_HOME/docs`, run: -In `$ZEPPELIN_HOME/docs`, run one of the desired commands: - -**Run locally** -``` -bundle exec jekyll serve --watch -``` - -**Run locally using docker** -``` +```bash docker run --rm -it \ - -v $PWD:/docs \ - -w /docs \ - -p '4000:4000' \ - ruby:3.3.5 \ - bash -c "bundle install && bundle exec jekyll serve --watch --host 0.0.0.0" + --user "$(id -u):$(id -g)" \ + -e HOME=/usr/local/bundle \ + -e BUNDLE_FROZEN=true \ + -v "$PWD:/docs" \ + -w /docs \ + -p '4000:4000' \ + ruby:4.0.6 \ + bash -lc "bundle install && bundle exec jekyll serve --watch --host 0.0.0.0" ``` -Using the above command, Jekyll will start a web server at `http://localhost:4000` and watch the `/docs` directory for updates. - - +Jekyll starts at `http://localhost:4000` and watches the `docs/` sources for +updates. The container runs with the current user's UID and GID so generated +files are not owned by `root` on the host. ## Contribute to Zeppelin documentation If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://zeppelin.apache.org/contribution/contributions.html). - ## For committers only ### Bumping up version in a new release - * `ZEPPELIN_VERSION` and `BASE_PATH` property in _config.yml +- Update `ZEPPELIN_VERSION` and `JB.BASE_PATH` in `_config.yml`. + +### Build versioned documentation -### Deploy to ASF svnpubsub infra - 1. generate static website in `./_site` +From `$ZEPPELIN_HOME/docs`, run: - ``` - # go to /docs under Zeppelin source - bundle exec jekyll build --safe - ``` +```bash +docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e HOME=/usr/local/bundle \ + -e BUNDLE_FROZEN=true \ + -v "$PWD:/docs" \ + -w /docs \ + ruby:4.0.6 \ + bash -lc "bundle install && bundle exec jekyll build --safe" +``` - 2. checkout ASF repo +Check the generated site for external resources and trackers: - ``` - svn co https://svn.apache.org/repos/asf/zeppelin asf-zeppelin - ``` +```bash +docker run --rm \ + -v "$PWD:/docs:ro" \ + -w /docs \ + ruby:4.0.6 \ + ruby check_external_resources.rb _site +``` - 3. copy `zeppelin/docs/_site` to `asf-zeppelin/site/docs/[VERSION]` - 4. `svn commit` +The generated site is written to `_site/`. Copy it to +`zeppelin-site/docs//` as part of the separate website publication +workflow. diff --git a/docs/_config.yml b/docs/_config.yml index 74b80adb9a3..eb2b9317025 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -2,11 +2,12 @@ # For more see: http://jekyllrb.com/docs/permalinks/ permalink: /:categories/:year/:month/:day/:title -exclude: [".rvmrc", ".rbenv-version", "README.md", "Rakefile", "changelog.md", "vendor", "node_modules", "scss"] +exclude: [".rvmrc", ".rbenv-version", "AGENTS.md", "Gemfile", "Gemfile.lock", "README.md", "Rakefile", "changelog.md", "check_external_resources.rb", "vendor", "node_modules", "scss"] highlighter: rouge -markdown: redcarpet -redcarpet: - extensions: ["tables"] +markdown: kramdown +kramdown: + input: GFM + show_warnings: true encoding: utf-8 # Themes are encouraged to use these universal variables @@ -18,8 +19,6 @@ author : name : The Apache Software Foundation email : dev@zeppelin.apache.org github : apache - twitter : ASF - feedburner : feedname ZEPPELIN_VERSION : 0.13.0-SNAPSHOT @@ -36,6 +35,7 @@ production_url : http://zeppelin.apache.org # JB : version : 0.3.0 + matomo_site_id : 69 # All links will be namespaced by BASE_PATH if defined. # Links in your website should always be prefixed with {{BASE_PATH}} @@ -77,53 +77,6 @@ JB : archive_path: /archive.html categories_path : /categories.html tags_path : /tags.html - atom_path : /atom.xml - rss_path : /rss.xml - - # Settings for comments helper - # Set 'provider' to the comment provider you want to use. - # Set 'provider' to false to turn commenting off globally. - # - comments : - provider : disqus - disqus : - short_name : jekyllbootstrap - livefyre : - site_id : 123 - intensedebate : - account : 123abc - facebook : - appid : 123 - num_posts: 5 - width: 580 - colorscheme: light - - # Settings for analytics helper - # Set 'provider' to the analytics provider you want to use. - # Set 'provider' to false to turn analytics off globally. - # - analytics : - provider : google_universal - google_classic : - tracking_id : 'UA-45176241-2' - google_universal : - tracking_id : 'UA-45176241-5' - domain : 'zeppelin.apache.org' - getclicky : - site_id : - mixpanel : - token : '_MIXPANEL_TOKEN_' - piwik : - baseURL : 'myserver.tld/piwik' # Piwik installation address (without protocol) - idsite : '1' # the id of the site on Piwik - - # Settings for sharing helper. - # Sharing is for things like tweet, plusone, like, reddit buttons etc. - # Set 'provider' to the sharing provider you want to use. - # Set 'provider' to false to turn sharing off globally. - # - sharing : - provider : false # Settings for all other include helpers can be defined by creating # a hash with key named for the given helper. ex: diff --git a/docs/_includes/JB/analytics b/docs/_includes/JB/analytics deleted file mode 100644 index 48d87c25fa0..00000000000 --- a/docs/_includes/JB/analytics +++ /dev/null @@ -1,18 +0,0 @@ -{% if site.safe and site.JB.analytics.provider and page.JB.analytics != false %} - -{% case site.JB.analytics.provider %} -{% when "google_classic" %} - {% include JB/analytics-providers/google-classic %} -{% when "google_universal" %} - {% include JB/analytics-providers/google-universal %} -{% when "getclicky" %} - {% include JB/analytics-providers/getclicky %} -{% when "mixpanel" %} - {% include JB/analytics-providers/mixpanel %} -{% when "piwik" %} - {% include JB/analytics-providers/piwik %} -{% when "custom" %} - {% include custom/analytics %} -{% endcase %} - -{% endif %} \ No newline at end of file diff --git a/docs/_includes/JB/analytics-providers/getclicky b/docs/_includes/JB/analytics-providers/getclicky deleted file mode 100644 index e9462f4f67f..00000000000 --- a/docs/_includes/JB/analytics-providers/getclicky +++ /dev/null @@ -1,12 +0,0 @@ - - diff --git a/docs/_includes/JB/analytics-providers/google-classic b/docs/_includes/JB/analytics-providers/google-classic deleted file mode 100644 index af099078a58..00000000000 --- a/docs/_includes/JB/analytics-providers/google-classic +++ /dev/null @@ -1,11 +0,0 @@ - \ No newline at end of file diff --git a/docs/_includes/JB/analytics-providers/google-universal b/docs/_includes/JB/analytics-providers/google-universal deleted file mode 100644 index dae744b994a..00000000000 --- a/docs/_includes/JB/analytics-providers/google-universal +++ /dev/null @@ -1,11 +0,0 @@ - \ No newline at end of file diff --git a/docs/_includes/JB/analytics-providers/mixpanel b/docs/_includes/JB/analytics-providers/mixpanel deleted file mode 100644 index 4406eb048d2..00000000000 --- a/docs/_includes/JB/analytics-providers/mixpanel +++ /dev/null @@ -1,11 +0,0 @@ - \ No newline at end of file diff --git a/docs/_includes/JB/analytics-providers/piwik b/docs/_includes/JB/analytics-providers/piwik deleted file mode 100755 index f016ed7ca4f..00000000000 --- a/docs/_includes/JB/analytics-providers/piwik +++ /dev/null @@ -1,10 +0,0 @@ - \ No newline at end of file diff --git a/docs/_includes/JB/comments b/docs/_includes/JB/comments deleted file mode 100644 index 4e9e600f6c0..00000000000 --- a/docs/_includes/JB/comments +++ /dev/null @@ -1,16 +0,0 @@ -{% if site.JB.comments.provider and page.comments != false %} - -{% case site.JB.comments.provider %} -{% when "disqus" %} - {% include JB/comments-providers/disqus %} -{% when "livefyre" %} - {% include JB/comments-providers/livefyre %} -{% when "intensedebate" %} - {% include JB/comments-providers/intensedebate %} -{% when "facebook" %} - {% include JB/comments-providers/facebook %} -{% when "custom" %} - {% include custom/comments %} -{% endcase %} - -{% endif %} \ No newline at end of file diff --git a/docs/_includes/JB/comments-providers/disqus b/docs/_includes/JB/comments-providers/disqus deleted file mode 100644 index 618a7b737bd..00000000000 --- a/docs/_includes/JB/comments-providers/disqus +++ /dev/null @@ -1,14 +0,0 @@ -
- - -blog comments powered by Disqus diff --git a/docs/_includes/JB/comments-providers/facebook b/docs/_includes/JB/comments-providers/facebook deleted file mode 100644 index 6b3e5e06921..00000000000 --- a/docs/_includes/JB/comments-providers/facebook +++ /dev/null @@ -1,9 +0,0 @@ -
- -
\ No newline at end of file diff --git a/docs/_includes/JB/comments-providers/intensedebate b/docs/_includes/JB/comments-providers/intensedebate deleted file mode 100644 index ab0c3c9769c..00000000000 --- a/docs/_includes/JB/comments-providers/intensedebate +++ /dev/null @@ -1,6 +0,0 @@ - - diff --git a/docs/_includes/JB/comments-providers/livefyre b/docs/_includes/JB/comments-providers/livefyre deleted file mode 100644 index 704b80392b1..00000000000 --- a/docs/_includes/JB/comments-providers/livefyre +++ /dev/null @@ -1,6 +0,0 @@ - - \ No newline at end of file diff --git a/docs/_includes/JB/matomo b/docs/_includes/JB/matomo new file mode 100644 index 00000000000..136bbb1b773 --- /dev/null +++ b/docs/_includes/JB/matomo @@ -0,0 +1,33 @@ + + + + + diff --git a/docs/_includes/JB/sharing b/docs/_includes/JB/sharing deleted file mode 100644 index f5b11518bd2..00000000000 --- a/docs/_includes/JB/sharing +++ /dev/null @@ -1,8 +0,0 @@ -{% if site.safe and site.JB.sharing.provider and page.JB.sharing != false %} - -{% case site.JB.sharing.provider %} -{% when "custom" %} - {% include custom/sharing %} -{% endcase %} - -{% endif %} \ No newline at end of file diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 9305359fdfb..9554b48a75c 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -37,7 +37,6 @@
  • Flink with Zeppelin
  • SQL with Zeppelin
  • Python with Zeppelin
  • -
  • R with Zeppelin
  • @@ -111,12 +110,12 @@
  • HTTP Security Headers
  • Notebook Storage
  • -
  • Git Storage
  • -
  • S3 Storage
  • -
  • Azure Storage
  • -
  • Google Cloud Storage
  • -
  • OSS Storage
  • -
  • MongoDB Storage
  • +
  • Git Storage
  • +
  • S3 Storage
  • +
  • Azure Storage
  • +
  • Google Cloud Storage
  • +
  • OSS Storage
  • +
  • MongoDB Storage
  • Operation
  • Configuration
  • @@ -137,9 +136,7 @@
  • Flink
  • JDBC
  • Python
  • -
  • R
  • -
  • Alluxio
  • BigQuery
  • Cassandra
  • Elasticsearch
  • diff --git a/docs/_includes/themes/zeppelin/default.html b/docs/_includes/themes/zeppelin/default.html index d2cd0719da3..36d88a36031 100644 --- a/docs/_includes/themes/zeppelin/default.html +++ b/docs/_includes/themes/zeppelin/default.html @@ -9,11 +9,6 @@ - - - @@ -37,9 +32,9 @@ - - - + {% if site.safe %} + {% include JB/matomo %} + {% endif %} @@ -54,6 +49,5 @@
    - {% include JB/analytics %} diff --git a/docs/_includes/themes/zeppelin/post.html b/docs/_includes/themes/zeppelin/post.html index 2774711aecb..14934d35b5b 100644 --- a/docs/_includes/themes/zeppelin/post.html +++ b/docs/_includes/themes/zeppelin/post.html @@ -43,7 +43,5 @@

    {{ page.title }} {% if page.tagline %}{{page.tagline}}{% endi {% endif %}

    -
    - {% include JB/comments %} diff --git a/docs/assets/themes/zeppelin/img/docs-img/edit_properties.png b/docs/assets/themes/zeppelin/img/docs-img/edit_properties.png index e67d49bcff4..4ab3c96396a 100644 Binary files a/docs/assets/themes/zeppelin/img/docs-img/edit_properties.png and b/docs/assets/themes/zeppelin/img/docs-img/edit_properties.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/ir_kernel.png b/docs/assets/themes/zeppelin/img/docs-img/ir_kernel.png deleted file mode 100644 index a1bf5ec188c..00000000000 Binary files a/docs/assets/themes/zeppelin/img/docs-img/ir_kernel.png and /dev/null differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/labeled-property-graph-model.svg b/docs/assets/themes/zeppelin/img/docs-img/labeled-property-graph-model.svg new file mode 100644 index 00000000000..0d2a53fa7f0 --- /dev/null +++ b/docs/assets/themes/zeppelin/img/docs-img/labeled-property-graph-model.svg @@ -0,0 +1,86 @@ + + + + Labeled property graph data model + People, notebooks, and a dataset represented as labeled nodes connected by typed relationships with properties. + + + + + + + + + + + + AUTHORED + since: 2024 + + + + VIEWED + date: Jul 18 + + + + READS + format: parquet + + + + READS + format: csv + + + + :Person:Author + name: Mina + team: Analytics + + + + :Person + name: Yun + team: Operations + + + + :Notebook + title: Flight Analysis + version: 3 + + + + :Notebook + title: Revenue Forecast + version: 7 + + + + :Dataset + name: Flight Records + owner: Data Platform + diff --git a/docs/assets/themes/zeppelin/img/docs-img/property-graph-example.svg b/docs/assets/themes/zeppelin/img/docs-img/property-graph-example.svg new file mode 100644 index 00000000000..7ae1a683dca --- /dev/null +++ b/docs/assets/themes/zeppelin/img/docs-img/property-graph-example.svg @@ -0,0 +1,87 @@ + + + + Property graph example + Four nodes connected by labeled, directed edges, with properties attached to both nodes and edges. + + + + + + + + + + + + KNOWS + + since: 2022 + + + + CREATED + + role: owner + + + + CONTRIBUTED + + commits: 18 + + + + READS + + format: parquet + + + name: Mina + age: 34 + + 1 + person + + + 2 + person + + name: Yun + + + title: Flight Analysis + language: Python + + 3 + notebook + + + 4 + dataset + + name: Flight Records + rows: 2.1M + diff --git a/docs/atom.xml b/docs/atom.xml deleted file mode 100644 index 7ec29339dd6..00000000000 --- a/docs/atom.xml +++ /dev/null @@ -1,28 +0,0 @@ ---- -layout: nil -title : ---- - - - - {{ site.title }} - - - {{ site.time | date_to_xmlschema }} - {{ site.production_url }} - - {{ site.author.name }} - {{ site.author.email }} - - - {% for post in site.posts %} - - {{ post.title }} - - {{ post.date | date_to_xmlschema }} - {{ site.production_url }}{{ post.id }} - {{ post.content | xml_escape }} - - {% endfor %} - - diff --git a/docs/check_external_resources.rb b/docs/check_external_resources.rb new file mode 100644 index 00000000000..563754687db --- /dev/null +++ b/docs/check_external_resources.rb @@ -0,0 +1,90 @@ +#!/usr/bin/env ruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "uri" + +SITE_DIR = ARGV.fetch(0, "_site") +RESOURCE_TAGS = %w[script img iframe link source video audio embed object].freeze +RESOURCE_PATTERN = + /<(#{RESOURCE_TAGS.join("|")})\b[^>]*\b(?:src|href|data)\s*=\s*["']([^"']+)["']/i +CSS_URL_PATTERN = /url\(\s*["']?([^"')]+)["']?\s*\)/i +TRACKER_PATTERN = + /google-analytics|googletag|gtag\s*\(|mixpanel|getclicky|piwik|disqus|connect\.facebook|(?:platform\.)?twitter\.com\/widgets/i +MATOMO_CODE_PATTERN = /matomo\.(?:js|php)|setTrackerUrl|\b_paq\b/i +ASF_MATOMO_URL = "https://analytics.apache.org/" + +def remote_host(value) + return nil unless value.match?(%r{\A(?:https?:)?//}i) + + normalized = value.start_with?("//") ? "https:#{value}" : value + URI.parse(normalized).host || :invalid +rescue URI::InvalidURIError + :invalid +end + +def asf_host?(host) + host != :invalid && (host == "apache.org" || host.end_with?(".apache.org")) +end + +violations = [] + +unless Dir.exist?(SITE_DIR) + warn "Site directory not found: #{SITE_DIR}" + exit 1 +end + +html_files = Dir.glob(File.join(SITE_DIR, "**", "*.html")).sort +if html_files.empty? + warn "No HTML files found in #{SITE_DIR}" + exit 1 +end + +html_files.each do |file| + File.read(file).scan(RESOURCE_PATTERN) do |tag, value| + host = remote_host(value) + next if host.nil? || asf_host?(host) + + violations << "#{file}: external #{tag} resource #{value}" + end +end + +Dir.glob(File.join(SITE_DIR, "**", "*.css")).sort.each do |file| + File.read(file).scan(CSS_URL_PATTERN) do |match| + value = match.first + host = remote_host(value) + next if host.nil? || asf_host?(host) + + violations << "#{file}: external CSS resource #{value}" + end +end + +Dir.glob(File.join(SITE_DIR, "**", "*.{html,js,css}")).sort.each do |file| + content = File.read(file) + violations << "#{file}: tracker or external embed code" if content.match?(TRACKER_PATTERN) + if content.match?(MATOMO_CODE_PATTERN) && !content.include?(ASF_MATOMO_URL) + violations << "#{file}: Matomo must use #{ASF_MATOMO_URL}" + end +end + +if violations.empty? + puts "No disallowed external resources or trackers found in #{SITE_DIR}" + exit 0 +end + +warn violations.join("\n") +exit 1 diff --git a/docs/development/contribution/how_to_contribute_code.md b/docs/development/contribution/how_to_contribute_code.md index 5cdf024710b..1464c339b35 100644 --- a/docs/development/contribution/how_to_contribute_code.md +++ b/docs/development/contribution/how_to_contribute_code.md @@ -140,14 +140,14 @@ Zeppelin has [set of integration tests](https://github.com/apache/zeppelin/tree/ ```bash TEST_SELENIUM=true ./mvnw test -Dtest=[TEST_NAME] -DfailIfNoTests=false \ --pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' +-pl 'zeppelin-interpreter,zeppelin-server' ``` For example, to run [ParagraphActionIT](https://github.com/apache/zeppelin/blob/master/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java), ```bash TEST_SELENIUM=true ./mvnw test -Dtest=ParagraphActionsIT -DfailIfNoTests=false \ --pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' +-pl 'zeppelin-interpreter,zeppelin-server' ``` You'll need Firefox web browser installed in your development environment. @@ -159,6 +159,6 @@ You can find issues for ) | Available option values used with the `option` widget | +|`optionValues` | `(Array)` | Available option values used with the `option` widget |
    @@ -278,4 +278,3 @@ class AwesomeVisualization extends Visualization { ... } ``` - diff --git a/docs/development/writing_zeppelin_interpreter.md b/docs/development/writing_zeppelin_interpreter.md index fa4970a293c..7f635bea49c 100644 --- a/docs/development/writing_zeppelin_interpreter.md +++ b/docs/development/writing_zeppelin_interpreter.md @@ -31,7 +31,7 @@ Interpreters in the same InterpreterGroup can reference each other. For example, -[InterpreterSetting](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java) is configuration of a given [InterpreterGroup](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java) and a unit of start/stop interpreter. +[InterpreterSetting](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java) is configuration of a given [InterpreterGroup](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java) and a unit of start/stop interpreter. All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via **[Thrift](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift)**. In 'Separate Interpreter(scoped / isolated) for each note' mode which you can see at the **Interpreter Setting** menu when you create a new interpreter, new interpreter instance will be created per note. But it still runs on the same JVM while they're in the same InterpreterSettings. @@ -159,7 +159,7 @@ To check out the list of languages supported, see the `mode-*.js` files under `z If you want to add a new set of syntax highlighting, -1. Add the `mode-*.js` file to [zeppelin-web/bower.json](https://github.com/apache/zeppelin/blob/master/zeppelin-web/bower.json) (when built, [zeppelin-web/src/index.html](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/index.html) will be changed automatically). +1. Add the `mode-*.js` file to [`zeppelin-web/bower.json`](https://github.com/apache/zeppelin/blob/master/zeppelin-web/bower.json) (when built, [`zeppelin-web/src/index.html`](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/index.html) will be changed automatically). 2. Add `language` field to `editor` object. Note that if you don't specify language field, your interpreter will use plain text mode for syntax highlighting. Let's say you want to set your language to `java`, then add: ```json diff --git a/docs/embedding-search.md b/docs/embedding-search.md new file mode 100644 index 00000000000..90fb266bb3e --- /dev/null +++ b/docs/embedding-search.md @@ -0,0 +1,224 @@ + + +# ZEPPELIN-6411: Semantic Search for Notebooks using Sentence Embeddings + +## Summary + +Add `EmbeddingSearch` — a new `SearchService` implementation that enables natural language +search across Zeppelin notebooks using ONNX-based sentence embeddings. This is a drop-in +replacement for `LuceneSearch` that understands meaning, not just keywords. + +**Example**: Searching "yesterday's spending" finds paragraphs containing +`SELECT sum(cost) FROM analytics.daily_sales WHERE date = current_date - interval '1' day` +— something keyword search cannot do (returns 0 results with LuceneSearch). + +## Motivation + +Zeppelin's current search (`LuceneSearch`) uses keyword-based full-text search with +Lucene's `StandardAnalyzer`. This has several limitations for notebook search: + +1. **No semantic understanding** — "yesterday's spend" won't find `current_date - 1` +2. **Poor SQL tokenization** — `StandardAnalyzer` breaks on underscores and dots in + table names like `analytics_db.daily_sales` +3. **No output indexing** — query results (table data, text output) are not searchable +4. **Exact match only** — users must guess the exact terms used in notebooks + +For teams with hundreds or thousands of notebooks (common in data/analytics teams), +finding the right query becomes a significant productivity bottleneck. + +## Architecture + +``` + SearchService (abstract) + ├── LuceneSearch (existing, keyword-based) + ├── EmbeddingSearch (new, semantic) + └── NoSearchService (existing, no-op) + +┌─────────────────────────────────────────────────────────────┐ +│ EmbeddingSearch │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ HuggingFace │ │ ONNX Runtime │ │ In-Memory Index │ │ +│ │ Tokenizer │→ │ Inference │→ │ float[][] + meta │ │ +│ │ (DJL) │ │ (CPU) │ │ ConcurrentHashMap│ │ +│ └──────────────┘ └──────────────┘ └────────┬─────────┘ │ +│ │ │ +│ Two-phase query: │ │ +│ 1. Embed query → cosine sim → find tables │ │ +│ 2. Re-rank with table boost → top-20 │ │ +│ ▼ │ +│ Index: text + title + output + tables embedding_index.bin│ +│ (persisted to disk, versioned) │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Model + +- **all-MiniLM-L6-v2**: 384-dimensional sentence embeddings +- 86MB ONNX model (quantized version available at 22MB) +- Downloaded on first use to `zeppelin.search.index.path/models/` +- Runs on CPU via ONNX Runtime (~5ms per paragraph) + +### Index + +- In-memory `ConcurrentHashMap` with `ReadWriteLock` +- Each entry stores: embedding (384 floats), notebook name, paragraph text, + title, extracted SQL table names, and paragraph output +- 10K paragraphs ≈ 15MB RAM, 50K paragraphs ≈ 75MB RAM +- Persisted as versioned binary file (`embedding_index.bin`, currently v3) +- Brute-force cosine similarity: < 50ms for 50K paragraphs + +### What gets indexed (vs. LuceneSearch) + +| Content | LuceneSearch | EmbeddingSearch | +|---------|:---:|:---:| +| Paragraph text | ✓ | ✓ | +| Paragraph title | ✓ | ✓ | +| Notebook name | ✓ | ✓ (in embedding context) | +| Paragraph output (TABLE, TEXT) | ✗ | ✓ | +| SQL table names (FROM/JOIN) | ✗ | ✓ (extracted + boosted) | +| Interpreter prefix stripped | ✗ | ✓ | + +### Two-Phase Search + +1. **Phase 1 — Table Discovery**: Run cosine similarity, collect SQL table names + from top-20 results weighted by rank +2. **Phase 2 — Table Boost**: Re-score results, boosting paragraphs that reference + the discovered tables (+0.05 per matching table) + +This helps queries like "click funnel analysis" surface all paragraphs that query +the same tables, even if their SQL text is very different. + +## Configuration + +Disabled by default. Enable with a single property: + +```xml + + + zeppelin.search.semantic.enable + true + +``` + +Requires `zeppelin.search.enable = true` (already the default). + +### Configuration matrix + +| `search.enable` | `search.semantic.enable` | Result | +|:---:|:---:|---| +| true | false (default) | LuceneSearch (existing behavior) | +| true | true | EmbeddingSearch (semantic) | +| false | any | NoSearchService | + +## Changes + +### New files +- `zeppelin-server/.../search/EmbeddingSearch.java` — Core implementation (~700 lines) +- `zeppelin-server/.../search/EmbeddingSearchTest.java` — 11 tests including semantic validation +- `docs/embedding-search.md` — This document + +### Modified files — Backend +- `zeppelin-server/pom.xml` — Add `onnxruntime` and `djl-tokenizers` dependencies +- `zeppelin-server/.../conf/ZeppelinConfiguration.java` — Add `ZEPPELIN_SEARCH_SEMANTIC_ENABLE` +- `zeppelin-server/.../server/ZeppelinServer.java` — Wire `EmbeddingSearch` based on config +- `NOTICE` — Attribution for ONNX Runtime and DJL + +### Modified files — Frontend +- `zeppelin-web-angular/.../result-item/` — Render search results with separate + code block, output block, and table name display (replaces Monaco editor) +- `zeppelin-web/src/app/search/` — Same improvements for Classic UI + +### Dependencies added +- `com.microsoft.onnxruntime:onnxruntime:1.18.0` (~50MB, Apache 2.0 compatible) +- `ai.djl.huggingface:tokenizers:0.28.0` (~2MB, Apache 2.0, JNA excluded to + avoid version conflict with Zeppelin's existing JNA 4.1.0) + +## Search Result Response Contract + +Both `LuceneSearch` and `EmbeddingSearch` return `List>` with +these keys: + +| Key | LuceneSearch | EmbeddingSearch | +|-----|-------------|-----------------| +| `id` | `noteId` or `noteId/paragraph/paragraphId` | Same | +| `name` | Notebook title | Notebook title | +| `snippet` | Highlighted paragraph text (`` tags) | Paragraph text (no highlighting) | +| `text` | Full paragraph text | Full paragraph text | +| `header` | Highlighted paragraph title (`` tags) | Paragraph title (plain) | +| `title` | Same as `header` | Paragraph title (plain) | +| `tables` | `""` (empty) | Space-separated SQL table names | +| `output` | `""` (empty) | Paragraph output (truncated to 300 chars) | + +The `title`, `tables`, and `output` fields are dedicated structured fields. The +`header` field preserves backward compatibility — for `LuceneSearch` it contains +the highlighted paragraph title, for `EmbeddingSearch` it contains the plain title. + +### Frontend Display + +Both Angular and Classic UIs render search results with: +- **Code block**: SQL/Python code with syntax-appropriate styling +- **Output block**: Paragraph execution results (from `output` field) +- **Table names**: Extracted SQL table names (from `tables` field) +- **Language badge**: `sql`, `python`, `md`, etc. + +## Design Decisions + +### Why ONNX Runtime instead of a Java ML library? + +ONNX Runtime is the standard inference engine for transformer models. It supports +the exact same model files used by Python (HuggingFace, ChromaDB, etc.), ensuring +embedding compatibility. + +### Why brute-force instead of HNSW/ANN? + +For Zeppelin's scale (typically < 50K paragraphs), brute-force cosine similarity +on normalized vectors is fast enough (< 50ms), exact (no approximation error), +and adds zero complexity. + +### Why download model on first use instead of bundling? + +The ONNX model is 86MB. Bundling it would bloat the Zeppelin distribution. +Downloading on first use keeps the distribution lean and allows users to swap models. + +### Why not use Lucene's vector search (since 9.0)? + +Zeppelin uses Lucene 8.7.0. Upgrading to 9.x is a separate, larger effort. + +## Testing + +```bash +# Run embedding search tests (requires model download, ~86MB first time) +ZEPPELIN_EMBEDDING_TEST=true mvn test -pl zeppelin-server \ + -Dtest=EmbeddingSearchTest + +# Run existing Lucene tests (should still pass, no changes) +mvn test -pl zeppelin-server -Dtest=LuceneSearchTest +``` + +### Key tests + +- `semanticSearchFindsRelatedConcepts` — validates that "yesterday's spending" + ranks a SQL spend query above an unrelated user count query +- `newParagraphIsLiveIndexed` — validates that newly added paragraphs are + immediately searchable without restart + +## Future Work + +- [ ] Quantized model support (22MB INT8 vs 86MB FP32) +- [ ] Hybrid search: combine embedding similarity with keyword matching +- [ ] Configurable model URL for air-gapped environments +- [ ] Batch embedding during initial index rebuild +- [ ] Similarity score display in search results diff --git a/docs/index.md b/docs/index.md index 75d362ca868..1393c0550d9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,10 +21,10 @@ limitations under the License.

    What is Apache Zeppelin?

    -

    +

    Multi-purpose notebook which supports

    -

    +

    20+ language backends