diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 0ff197d01..29d19993a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,18 +41,23 @@ env: jobs: - # Build documentation handling warnings as errors. - # If success, upload built docs. If failure, notify telegram and upload logs. + # Build documentation handling warnings as errors in both HTML and PDF. + # If success, upload built docs as artifact. + # If failure in HTML, notify telegram and upload logs. build: - name: Build translated docs + name: Build docs runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + format: [ html, latex, epub ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 5 - name: Set up Python 3 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ inputs.version }} cache: pip @@ -65,25 +70,22 @@ jobs: - name: setup run: ./scripts/setup.sh + - name: Install APT dependencies + if: matrix.format == 'latex' + run: sudo apt update -y && sudo apt install librsvg2-bin -y + - name: Add problem matcher uses: sphinx-doc/github-problem-matcher@v1.1 - name: Build docs id: build - run: ./scripts/build.sh - - - name: Upload artifact - docs - if: steps.build.outcome == 'success' - uses: actions/upload-artifact@v4.3.5 - with: - name: docs - path: cpython/Doc/build/html + run: ./scripts/build.sh ${{ matrix.format }} - name: Prepare notification (only on error) - if: always() && steps.build.outcome == 'failure' + if: always() && steps.build.outcome == 'failure' && matrix.format == 'html' id: prepare run: | - scripts/prepmsg.sh logs/sphinxwarnings.txt logs/notify.txt + scripts/prepmsg.sh logs/sphinxwarnings-${{ matrix.format }}.txt logs/notify.txt cat logs/notify.txt env: GITHUB_JOB: ${{ github.job }} @@ -101,11 +103,35 @@ jobs: - name: Upload artifact - log files if: always() && steps.build.outcome == 'failure' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: ${{ inputs.version }}-build-logs + name: logs-${{ inputs.version }}-${{ matrix.format }} path: logs/* + + - name: Upload artifact - docs + if: always() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v7 + with: + name: python-docs-pt-br-${{ inputs.version }}-${{ matrix.format }} + path: cpython/Doc/build/${{ matrix.format }} + # Build Python docs in PDF format and make available for download. + output-pdf: + name: Build docs (pdf) + runs-on: ubuntu-latest + needs: [ 'build' ] + steps: + - uses: actions/download-artifact@v8 + with: + name: python-docs-pt-br-${{ inputs.version }}-latex + - run: sudo apt-get update + - run: sudo apt-get install -y latexmk texlive-xetex fonts-freefont-otf xindy texlive-lang-portuguese + - run: make + - uses: actions/upload-artifact@v7 + if: always() + with: + name: python-docs-pt-br-${{ inputs.version }}-pdf + path: ./*.pdf # Run sphinx-lint to find wrong reST syntax in PO files. Always store logs. # If issues are found, notify telegram and upload logs. @@ -113,12 +139,12 @@ jobs: name: Lint translations runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 5 - name: Set up Python 3 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ inputs.version }} cache: pip @@ -160,7 +186,33 @@ jobs: - name: Upload artifact - log files if: always() && steps.lint.outcome == 'failure' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ inputs.version }}-lint-logs path: logs/* + + + # Check for zero-width space charcters in translations. + # These are known to come together with (some?) machine translation like Google translate, + # and - as one of the consequences - it may avoid Transifex glossary matching (e.g. variáveis) + zero-width-space: + name: Check for zero-width space characters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.version }} + + - name: Make sure the repository is up to date + if: github.event_name != 'pull_request' + run: git pull --rebase + + - name: Remove zero-width space characters + run: | + sed -i 's/\xe2\x80\x8b//g' *.po **/*.po + # Undo removal from where we should not + sed -i 's|`````|``​`​``|' reference/lexical_analysis.po + + - name: Show difference (error if there is any) + run: | + git diff --exit-code --color-words diff --git a/.github/workflows/python-310.yml b/.github/workflows/python-310.yml index da24a35e1..528e1d252 100644 --- a/.github/workflows/python-310.yml +++ b/.github/workflows/python-310.yml @@ -3,7 +3,7 @@ name: python-310 on: workflow_dispatch: schedule: - - cron: '45 23 * * *' + - cron: '0 0 * * *' jobs: sync: diff --git a/.github/workflows/python-311.yml b/.github/workflows/python-311.yml index ec56ea095..d94ec6867 100644 --- a/.github/workflows/python-311.yml +++ b/.github/workflows/python-311.yml @@ -3,7 +3,7 @@ name: python-311 on: workflow_dispatch: schedule: - - cron: '30 23 * * *' + - cron: '45 23 * * *' jobs: sync: diff --git a/.github/workflows/python-312.yml b/.github/workflows/python-312.yml index b7f747c44..84e24aa4d 100644 --- a/.github/workflows/python-312.yml +++ b/.github/workflows/python-312.yml @@ -2,14 +2,8 @@ name: python-312 on: workflow_dispatch: - pull_request: - branches: - - '3.12' - push: - branches: - - '3.12' schedule: - - cron: '15 23 * * *' + - cron: '30 23 * * *' jobs: sync: diff --git a/.github/workflows/python-313.yml b/.github/workflows/python-313.yml index 60cd85f37..65672876b 100644 --- a/.github/workflows/python-313.yml +++ b/.github/workflows/python-313.yml @@ -2,6 +2,8 @@ name: python-313 on: workflow_dispatch: + schedule: + - cron: '15 23 * * *' pull_request: branches: - main @@ -10,20 +12,18 @@ on: branches: - main - '3.13' - schedule: - - cron: '0 23 * * *' jobs: sync: uses: ./.github/workflows/sync.yml with: - tx_project: python-newest + tx_project: ${{ github.workflow }} version: 3.13 secrets: inherit check: uses: ./.github/workflows/check.yml needs: sync with: - tx_project: python-newest + tx_project: ${{ github.workflow }} version: 3.13 secrets: inherit diff --git a/.github/workflows/python-314.yml b/.github/workflows/python-314.yml new file mode 100644 index 000000000..000d02254 --- /dev/null +++ b/.github/workflows/python-314.yml @@ -0,0 +1,29 @@ +name: python-314 + +on: + workflow_dispatch: + schedule: + - cron: '0 23 * * *' + pull_request: + branches: + - main + - '3.14' + push: + branches: + - main + - '3.14' + +jobs: + sync: + uses: ./.github/workflows/sync.yml + with: + tx_project: python-newest + version: 3.14 + secrets: inherit + check: + uses: ./.github/workflows/check.yml + needs: sync + with: + tx_project: python-newest + version: 3.14 + secrets: inherit diff --git a/.github/workflows/python-39.yml b/.github/workflows/python-39.yml index 8dd0a67ad..a49ac9dbf 100644 --- a/.github/workflows/python-39.yml +++ b/.github/workflows/python-39.yml @@ -2,8 +2,6 @@ name: python-39 on: workflow_dispatch: - schedule: - - cron: '0 0 * * *' jobs: sync: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 034084f00..3a7f526af 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -30,43 +30,24 @@ jobs: # 1- Set up environment - name: Check out this repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Checkout CPython ${{ env.PYDOC_VERSION }} - uses: actions/checkout@v4 - with: - repository: 'python/cpython' - ref: ${{ env.PYDOC_VERSION }} - path: cpython - - name: Set language dir variable run: - echo "LANGUAGE_DIR=cpython/Doc/locales/${{ env.PYDOC_LANGUAGE }}/LC_MESSAGES" >> $GITHUB_ENV + echo "PYDOC_LANG_DIR=${{ env.PYDOC_VERSION }}" >> $GITHUB_ENV - name: Checkout this repository ${{ env.PYDOC_VERSION }} - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: ${{ env.PYDOC_VERSION }} - path: ${{ env.LANGUAGE_DIR }} + path: ${{ env.PYDOC_LANG_DIR }} - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ inputs.version }} allow-prereleases: true cache: 'pip' - cache-dependency-path: | - requirements.txt - cpython/Doc/requirements.txt - - - name: Check for Transifex API Token availability - id: secret-check - # perform secret check & put boolean result as an output - shell: bash - run: | - available=false - [[ "${{ secrets.TX_TOKEN }}" != '' ]] && available=true - echo "available=$available" >> $GITHUB_OUTPUT - echo "available=$available" + pip-install: -r requirements.txt # 2- Install dependencies @@ -78,22 +59,20 @@ jobs: - name: Install APT dependencies run: sudo apt update -y && sudo apt install gettext -y - - name: Install Python dependencies - run: | - pip install -r requirements.txt - make -C cpython/Doc venv - # 3- Pull translations - - name: Generate template files and Transifex config file - run: ./scripts/generate_templates.sh + - name: Generate updated .tx/config + if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} + run: python scripts/generate_txconfig.py -p ./${{ env.PYDOC_LANG_DIR }} ${{ env.PYDOC_TX_PROJECT }} + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} - name: Pull translations from Transifex id: pull if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} run: | # Clean up obsolete files - find ./${{ env.LANGUAGE_DIR }} -name '*.po' -exec rm {} \; + find ./${{ env.PYDOC_LANG_DIR }} -name '*.po' -exec rm {} \; ./scripts/pull_translations.sh env: TX_TOKEN: ${{ secrets.TX_TOKEN }} @@ -103,38 +82,42 @@ jobs: run: | newer_branch=${PYDOC_VERSION%%.*}.$((${PYDOC_VERSION##*.}+1)) git clone --depth 1 --single-branch --branch $newer_branch https://github.com/python/python-docs-pt-br ${newer_branch}-dir - pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.LANGUAGE_DIR }}/{**/,}*.po + pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.PYDOC_LANG_DIR }}/{**/,}*.po rm -rf ./${newer_branch}-dir - name: powrap if: steps.pull.outcome == 'success' run: | - cd ./${{ env.LANGUAGE_DIR }} + cd ./${{ env.PYDOC_LANG_DIR }} powrap *.po **/*.po - name: Update statistics - if: always() && steps.secret-check.outputs.available == 'true' + if: always() run: | - python ./scripts/tx_stats.py > ./${{ env.LANGUAGE_DIR }}/stats.json - git -C ./${{ env.LANGUAGE_DIR }} diff stats.json + ./scripts/stats.py + git -C ./${{ env.PYDOC_LANG_DIR }} diff stats.json env: - TX_TOKEN: ${{ secrets.TX_TOKEN }} + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} - name: Update potodo.md if: always() run: | ./scripts/potodo.sh - git diff ./${{ env.LANGUAGE_DIR }}/potodo.md + git -C ./${{ env.PYDOC_LANG_DIR }} diff potodo.md + env: + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} # 4- Commit and push translations - name: Commit run: ./scripts/commit.sh + env: + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} - name: Push if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} run: | - cd ./${{ env.LANGUAGE_DIR }} + cd ./${{ env.PYDOC_LANG_DIR }} git push diff --git a/README.rst b/README.rst index c114af9ee..548034df7 100644 --- a/README.rst +++ b/README.rst @@ -26,51 +26,51 @@ Maintained versions: - Sync status - Translation progress - Total strings + * - `3.14 `_ + - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-314/badge.svg + :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-314 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.14%2Fstats.json&query=completion&label=pt_BR + :alt: Brazilian Portuguese translation status for Python 3.14 + :target: https://app.transifex.com/python-doc/python-newest/ + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.14%2Fstats.json&query=entries&label=3.14 + :alt: Total strings for Python 3.14 + :target: https://app.transifex.com/python-doc/python-newest/ * - `3.13 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-313/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-313 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.13%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.13%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.13 :target: https://app.transifex.com/python-doc/python-newest/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.13%2Fstats.json&query=total&label=3.13 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.13%2Fstats.json&query=entries&label=3.13 :alt: Total strings for Python 3.13 :target: https://app.transifex.com/python-doc/python-newest/ * - `3.12 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-312/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-312 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.12%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.12%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.12 :target: https://app.transifex.com/python-doc/python-312/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.12%2Fstats.json&query=total&label=3.12 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.12%2Fstats.json&query=entries&label=3.12 :alt: Total strings for Python 3.12 :target: https://app.transifex.com/python-doc/python-312/ * - `3.11 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-311/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-311 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.11%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.11%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.11 :target: https://app.transifex.com/python-doc/python-311/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.11%2Fstats.json&query=total&label=3.11 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.11%2Fstats.json&query=entries&label=3.11 :alt: Total strings for Python 3.11 :target: https://app.transifex.com/python-doc/python-311/ * - `3.10 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-310/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-310 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.10%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.10%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.10 :target: https://app.transifex.com/python-doc/python-310/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.10%2Fstats.json&query=total&label=3.10 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.10%2Fstats.json&query=entries&label=3.10 :alt: Total strings for Python 3.10 :target: https://app.transifex.com/python-doc/python-310/ - * - `3.9 `_ - - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-39/badge.svg - :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-39 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.9%2Fstats.json&query=translation&label=pt_BR - :alt: Brazilian Portuguese translation status for Python 3.9 - :target: https://app.transifex.com/python-doc/python-39/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.9%2Fstats.json&query=total&label=3.9 - :alt: Total strings for Python 3.9 - :target: https://app.transifex.com/python-doc/python-39/ EOL versions: @@ -84,22 +84,31 @@ EOL versions: - Sync status - Translation progress - Total strings + * - `3.9 `_ + - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-39/badge.svg + :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-39 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.9%2Fstats.json&query=completion&label=pt_BR + :alt: Brazilian Portuguese translation status for Python 3.9 + :target: https://app.transifex.com/python-doc/python-39/ + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.9%2Fstats.json&query=entries&label=3.9 + :alt: Total strings for Python 3.9 + :target: https://app.transifex.com/python-doc/python-39/ * - `3.8 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-38/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-38 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.8%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.8%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.8 :target: https://app.transifex.com/python-doc/python-38/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.8%2Fstats.json&query=total&label=3.8 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.8%2Fstats.json&query=entries&label=3.8 :alt: Total strings for Python 3.8 :target: https://app.transifex.com/python-doc/python-38/ * - `3.7 `_ - .. image:: https://github.com/python/python-docs-pt-br/workflows/python-37/badge.svg :target: https://github.com/python/python-docs-pt-br/actions?workflow=python-37 - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.7%2Fstats.json&query=translation&label=pt_BR + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.7%2Fstats.json&query=completion&label=pt_BR :alt: Brazilian Portuguese translation status for Python 3.7 :target: https://app.transifex.com/python-doc/python-37/ - - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.7%2Fstats.json&query=total&label=3.7 + - .. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgithub.com%2Fpython%2Fpython-docs-pt-br%2Fraw%2F3.7%2Fstats.json&query=entries&label=3.7 :alt: Total strings for Python 3.7 :target: https://app.transifex.com/python-doc/python-37/ diff --git a/requirements.txt b/requirements.txt index 0516f31ba..23aaf5bae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -pomerge==0.1.4 -potodo==0.22.0 +polib==1.2.0 +pomerge==0.2.1 +potodo==0.35 powrap==1.0.2 -sphinx-intl==2.3.1 -sphinx-lint==1.0.0 +sphinx-intl==2.3.2 +sphinx-lint==1.0.2 diff --git a/scripts/build.sh b/scripts/build.sh index 6be5c8028..1e01af409 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,10 +1,17 @@ #!/bin/sh -# Build translated docs to pop up errors +# Build translated docs +# Expects input like 'html' and 'latex', defaults to 'html'. # # SPDX-License-Identifier: CC0-1.0 set -xeu +if [ -z "$1" ]; then + format=html +else + format="$1" +fi + # Fail earlier if required variables are not set test -n ${PYDOC_LANGUAGE+x} @@ -14,15 +21,15 @@ mkdir -p logs # If version is 3.12 or older, set gettext_compact. # This confval is not needed since 3.12. # In 3.13, its presence messes 3.13's syntax checking (?) -opts="-D language=${PYDOC_LANGUAGE} --keep-going -w ../../logs/sphinxwarnings.txt" -minor_version=$(git branch --show-current | sed 's|^3\.||') +opts="-D language=${PYDOC_LANGUAGE} --keep-going -w ../../logs/sphinxwarnings-${format}.txt" +minor_version=$(git -C cpython/Doc branch --show-current | sed 's|^3\.||') if [ $minor_version -lt 12 ]; then - opts += '-D gettext_compact=False' + opts="$opts -D gettext_compact=False" fi -make -C cpython/Doc html SPHINXOPTS="${opts}" +make -C cpython/Doc "${format}" SPHINXOPTS="${opts}" # Remove empty file -if [ ! -s logs/sphinxwarnings.txt ]; then - rm logs/sphinxwarnings.txt +if [ ! -s "logs/sphinxwarnings-${format}.txt" ]; then + rm "logs/sphinxwarnings-${format}.txt" fi diff --git a/scripts/commit.sh b/scripts/commit.sh index fb1b752e2..a5c5d742d 100755 --- a/scripts/commit.sh +++ b/scripts/commit.sh @@ -5,7 +5,12 @@ set -eu -cd $(dirname $0)/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES +test -n ${PYDOC_LANGUAGE+x} + +rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" + +cd "$language_dir" extra_files=".tx/config stats.json potodo.md" @@ -19,8 +24,8 @@ set -u # Set for removal the deleted obsolete PO files git status -s | grep '^ D ' | cut -d' ' -f3 | xargs -r git rm -# Add only updates that do not consist only of 'POT-Creation-Date' header change -git diff -I'^"POT-Creation-Date: ' -I'^"Language: pt_BR' --numstat *.po **/*.po | cut -f3 | xargs -r git add -v +# Add only updates that do not consist only of the following header lines +git diff -I'^# Copyright ' -I'^"Project-Id-Version: ' -I'^"POT-Creation-Date: ' -I'^"Language: ' --numstat *.po **/*.po | cut -f3 | xargs -r git add -v # Add currently untracked PO files, and update other helper files untracked_files=$(git ls-files -o --exclude-standard *.po **/*.po) @@ -41,4 +46,7 @@ fi set -u # Commit only if there is any cached file -git diff-index --cached --quiet HEAD || { git add -v $extra_files; git commit -vm "Update translations"; } +if ! git diff-index --cached --quiet HEAD; then + git add -v $extra_files + git commit -vm "$($rootdir/scripts/generate_commit_msg.py)" +fi diff --git a/scripts/generate_commit_msg.py b/scripts/generate_commit_msg.py new file mode 100755 index 000000000..504e24e3c --- /dev/null +++ b/scripts/generate_commit_msg.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +""" +Generate a commit message +Parses staged files and generates a commit message with Last-Translator's as +co-authors. +Based on Stan Ulbrych's implementation for Python Doc' Polish team +""" + +import argparse +import contextlib +import os +from subprocess import run, CalledProcessError +from pathlib import Path + +from polib import pofile, POFile + + +def generate_commit_msg(): + translators: set[str] = set() + + result = run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"], + capture_output=True, + text=True, + check=True, + ) + staged = [ + filename for filename in result.stdout.splitlines() if filename.endswith(".po") + ] + + for file in staged: + staged_file = run( + ["git", "show", f":{file}"], capture_output=True, text=True, check=True + ).stdout + try: + old_file = run( + ["git", "show", f"HEAD:{file}"], + capture_output=True, + text=True, + check=True, + ).stdout + except CalledProcessError: + old_file = "" + + new_po = pofile(staged_file) + old_po = pofile(old_file) if old_file else POFile() + old_entries = {entry.msgid: entry.msgstr for entry in old_po} + + for entry in new_po: + if entry.msgstr and ( + entry.msgid not in old_entries + or old_entries[entry.msgid] != entry.msgstr + ): + # Prevent failure on missing Last-Translator field. + # Transifex only adds Last-Translator if someone from + # the team translated. If it was uploaded by an account + # that is not in the team, this field will be missing. + translator = ( + (new_po.metadata.get("Last-Translator") or "").split(",")[0].strip() + ) + if translator: + translators.add(f"Co-Authored-By: {translator}") + break + + print("Update translation\n\n" + "\n".join(translators)) + + +# contextlib implemented chdir since Python 3.11 +@contextlib.contextmanager +def chdir(path: Path): + """Temporarily change the working directory.""" + original_dir = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(original_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate commit message with translators as co-authors." + ) + parser.add_argument( + "path", + type=Path, + nargs="?", + default=".", + help="Path to the Git repository (default: current directory)", + ) + args = parser.parse_args() + + with chdir(args.path): + generate_commit_msg() diff --git a/scripts/generate_txconfig.py b/scripts/generate_txconfig.py new file mode 100755 index 000000000..727f9b5a4 --- /dev/null +++ b/scripts/generate_txconfig.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Generate the .tx/config file based on the existing projects +in Python docs Transifex project. Takes an project slug as +positional argument, e.g. python-newest or python-313 +""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +# Replaces required to fix the default values set by 'tx add remote' command. +# Add or remove +TEXT_TO_REPLACE = { + "2_": "2.", + "3_": "3.", + "glossary_": "glossary", + "collections_": "collections.", + "compression_": "compression.", + "concurrent_": "concurrent.", + "curses_": "curses.", + "email_": "email.", + "html_": "html.", + "http_": "http.", + "importlib_resources_": "importlib.resources.", + "importlib_": "importlib.", + "logging_": "logging.", + "multiprocessing_": "multiprocessing.", + "os_": "os.", + "string_": "string.", + "sys_monitoring": "sys.monitoring", + "tkinter_": "tkinter.", + "unittest_": "unittest.", + "urllib_": "urllib.", + "xml_dom_": "xml.dom.", + "xml_etree_": "xml.etree.", + "xmlrpc_": "xmlrpc.", + "xml_sax_": "xml.sax.", + "xml_": "xml.", +} + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root-path", + "-p", + default=Path("."), + help="Path to the translation files, and also the .tx/config (defaults to current directory)", + ) + parser.add_argument( + "tx_project", help="Slug of the Transifex project to query resources from" + ) + return parser.parse_args() + + +def reset_tx_config(txconfig: Path): + """Create or reset the .tx/config file with basic header.""" + txconfig.parent.mkdir(exist_ok=True) + txconfig.write_text("[main]\nhost = https://www.transifex.com\n", encoding="utf-8") + print("Initialized .tx/config.") + + +def populate_resources_from_remote(config_file: Path, tx_project: str): + """Add the remote resources from the Transifex project to .tx/config.""" + result = subprocess.run( + [ + "tx", + "--config", + str(config_file), + "add", + "remote", + "--file-filter", + "/.", + f"https://app.transifex.com/python-doc/{tx_project}/", + ], + check=True, + ) + if result.returncode != 0: + print("Failed to add the resources from remote:") + print(result.stderr.strip()) + sys.exit(result.returncode) + print("Added remote resources to Transifex.") + + +def patch_config(txconfig: Path): + """Patch .tx/config to fixing PO filenames to match the expected.""" + content = txconfig.read_text(encoding="utf-8").splitlines() + new_lines = [] + + for line in content: + if line.startswith(("source_file", "source_lang")): + continue + + if line.startswith("file_filter"): + line = line.replace("/", "") + line = line.replace("--", "/") + + for pattern, replacement in TEXT_TO_REPLACE.items(): + if pattern in line: + line = line.replace(pattern, replacement) + break + + new_lines.append(line) + + text = "\n".join(new_lines) + + txconfig.write_text(text + "\n", encoding="utf-8") + print("Updated .tx/config with character substitutions") + + +def main(): + args = parse_args() + config_path = Path(".tx/config") + if args.root_path: + config_path = args.root_path / config_path + reset_tx_config(config_path) + populate_resources_from_remote(config_path, args.tx_project) + patch_config(config_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint.sh b/scripts/lint.sh index 721790e5d..82ba7def1 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -20,14 +20,10 @@ touch logs/sphinxlint.txt cd cpython/Doc -# If version is 3.12 or newer, then generate POT disabling literal blocks and -# update translations with fresh POT files. If version 3.11 or older, -# disable new 'unnecessary-parentheses' check, not fixed before these versions. +# If version 3.11 or older, disable new 'unnecessary-parentheses' check, +# not fixed before 3.12. minor_version=$(git branch --show-current | sed 's|^3\.||') -if [ $minor_version -ge 12 ]; then - make gettext SPHINXOPTS='-q -Dgettext_additional_targets=["index"]' - sphinx-intl update -p build/gettext -l ${PYDOC_LANGUAGE} > /dev/null -else +if [ $minor_version -le 11 ]; then alias sphinx-lint='sphinx-lint --disable unnecessary-parentheses' fi @@ -36,9 +32,6 @@ set +e sphinx-lint 2> $(realpath "$rootdir/logs/sphinxlint.txt") set -e -# Undo changes to undo literal blocks disabling -git checkout . - cd "$rootdir" # Check of logfile is empty diff --git a/scripts/potodo.sh b/scripts/potodo.sh index 0d7e1e2c2..6b343e0da 100755 --- a/scripts/potodo.sh +++ b/scripts/potodo.sh @@ -10,8 +10,9 @@ test -n ${PYDOC_VERSION+x} test -n ${PYDOC_LANGUAGE+x} rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" -cd "$rootdir"/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES +cd "$language_dir" potodo --no-cache > potodo.md diff --git a/scripts/pull_translations.sh b/scripts/pull_translations.sh index 565046c90..a85d79b69 100755 --- a/scripts/pull_translations.sh +++ b/scripts/pull_translations.sh @@ -20,8 +20,12 @@ set -xeu test -n ${PYDOC_TX_PROJECT+x} +test -n ${PYDOC_LANGUAGE+x} -cd "$(dirname $0)/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES/" +rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" + +cd "$language_dir" # If a PO file is provided as input, convert it into Transifex resource # and add it be pulled (instead of pulling all translations files). @@ -41,3 +45,9 @@ if [ $# -gt 0 ]; then fi tx pull -f -l "${PYDOC_LANGUAGE}" ${resources_to_pull} + +# Drop translation of Python's changelog in python 3.12 or older. +minor_version=$(git branch --show-current | sed 's|^3\.||') +if [ $minor_version -le 12 ]; then + git checkout whatsnew/changelog.po +fi diff --git a/scripts/stats.py b/scripts/stats.py new file mode 100755 index 000000000..4dde51b18 --- /dev/null +++ b/scripts/stats.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Obtain translation stats from the PO files directory and +store it with JSON format into 'stats.json'. +""" + +import json +import os +import logging +from datetime import datetime, timezone +from pathlib import Path + +from potodo.po_file import PoDirectory + +logging.basicConfig(level=logging.INFO) + + +def main() -> None: + """Main function to generate translation stats.""" + + lang_dir = os.environ.get("PYDOC_LANG_DIR") + if lang_dir: + pofiles_path = Path(lang_dir) + else: + language = os.environ.get("PYDOC_LANGUAGE") + if not language: + raise ValueError("Environment variable PYDOC_LANGUAGE is not set.") + pofiles_path = Path(f"cpython/Doc/locales/{language}/LC_MESSAGES") + + if not pofiles_path.exists(): + raise FileNotFoundError(f"Path does not exist: {pofiles_path}") + + # Check for PO files inside the pofiles_path + if not list(pofiles_path.rglob("*.po")): + raise FileNotFoundError(f"No PO files found in {pofiles_path}") + + stats = PoDirectory(pofiles_path, use_cache=False) + stats.scan() + + stats_data = { + "completion": str(round(stats.completion, 2)) + "%", + "translated": stats.translated, + "entries": stats.entries, + "updated_at": datetime.now(timezone.utc).isoformat(timespec="seconds") + "Z", + } + + stats_json = pofiles_path / "stats.json" + try: + with stats_json.open("w") as output_file: + json.dump(stats_data, output_file) + logging.info(f"Content written to {stats_json}") + except IOError as e: + logging.error(f"Failed to write to {stats_json}: {e}") + raise + + +if __name__ == "__main__": + try: + main() + except Exception as e: + logging.error(f"An error occurred: {e}")