diff --git a/.github/actions/matrixai-env-setup/action.yml b/.github/actions/matrixai-env-setup/action.yml new file mode 100644 index 0000000..e10d5ce --- /dev/null +++ b/.github/actions/matrixai-env-setup/action.yml @@ -0,0 +1,65 @@ +name: 'MatrixAI Environment Setup' +description: 'Installs Nix on GitHub Actions for the supported platforms: Linux and macOS.' +inputs: + extra_nix_config: + description: 'Gets appended to `/etc/nix/nix.conf` if passed.' + github_access_token: + description: 'Configure nix to pull from github using the given github token.' + install_url: + description: 'Installation URL that will contain a script to install Nix.' + install_options: + description: 'Additional installer flags passed to the installer script.' + nix_path: + description: 'Set NIX_PATH environment variable.' + enable_kvm: + description: 'Enable KVM for hardware-accelerated virtualization on Linux, if available.' + required: false + default: true +runs: + using: 'composite' + steps: + - run: | + ${GITHUB_ACTION_PATH}/matrixai-env-setup.sh + shell: bash + env: + INPUT_EXTRA_NIX_CONFIG: | + ${{ inputs.extra_nix_config }} + substituters = s3://matrix-ai-nix-cache?profile=matrix-nix-cache®ion=ap-southeast-2 https://cache.nixos.org/ + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= matrix-ai-nix-cache:yhxzASVutUGCY2o/U4jkiNVj06M6Fi1h94LiC5TkYBg= + INPUT_REGISTRY: | + { + "flakes": [ + { + "exact": true, + "from": { "id": "nixpkgs-matrix", "type": "indirect" }, + "to": { + "type": "git", + "url": "https://github.com/MatrixAI/nixpkgs-matrix" + } + }, + { + "exact": true, + "from": { "id": "nixpkgs-matrix-private", "type": "indirect" }, + "to": { + "type": "git", + "url": "https://github.com/MatrixAI/nixpkgs-matrix-private" + } + } + ], + "version": 2 + } + INPUT_GITHUB_ACCESS_TOKEN: ${{ inputs.github_access_token }} + INPUT_INSTALL_OPTIONS: ${{ inputs.install_options }} + INPUT_INSTALL_URL: ${{ inputs.install_url }} + INPUT_NIX_PATH: ${{ inputs.nix_path }} + INPUT_ENABLE_KVM: ${{ inputs.enable_kvm }} + GITHUB_TOKEN: ${{ github.token }} + - run: | + nix profile install nixpkgs-matrix#cacert nixpkgs-matrix#tzdata nixpkgs-matrix#polykey-cli + TZDATA=$(nix eval --raw nixpkgs-matrix#tzdata.outPath) + CACERT=$(nix eval --raw nixpkgs-matrix#cacert.outPath) + echo "TZDIR=$TZDATA/share/zoneinfo" >> "$GITHUB_ENV" + echo "GIT_SSL_CAINFO=$CACERT/etc/ssl/certs/ca-bundle.crt" >> "$GITHUB_ENV" + echo "NIX_SSL_CERT_FILE=$CACERT/etc/ssl/certs/ca-bundle.crt" >> "$GITHUB_ENV" + shell: bash + diff --git a/.github/actions/matrixai-env-setup/matrixai-env-setup.sh b/.github/actions/matrixai-env-setup/matrixai-env-setup.sh new file mode 100755 index 0000000..675cc79 --- /dev/null +++ b/.github/actions/matrixai-env-setup/matrixai-env-setup.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +if nix_path="$(type -p nix)" ; then + echo "Aborting: Nix is already installed at ${nix_path}" + exit +fi + +if [[ ($OSTYPE =~ linux) && ($INPUT_ENABLE_KVM == 'true') ]]; then + enable_kvm() { + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-matrixai-env-setup-action-kvm.rules + sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm + } + + echo '::group::Enabling KVM support' + enable_kvm && echo 'Enabled KVM' || echo 'KVM is not available' + echo '::endgroup::' +fi + +# GitHub command to put the following log messages into a group which is collapsed by default +echo "::group::Installing Nix" + +# Create a temporary workdir +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +# Configure Nix +add_config() { + echo "$1" >> "$workdir/nix.conf" +} +add_config "show-trace = true" +# Set jobs to number of cores +add_config "max-jobs = auto" +if [[ $OSTYPE =~ darwin ]]; then + add_config "ssl-cert-file = /etc/ssl/cert.pem" +fi +# Allow binary caches for user +add_config "trusted-users = root ${USER:-}" +# Add a GitHub access token. +# Token-less access is subject to lower rate limits. +if [[ -n "${INPUT_GITHUB_ACCESS_TOKEN:-}" ]]; then + echo "::debug::Using the provided github_access_token for github.com" + add_config "access-tokens = github.com=$INPUT_GITHUB_ACCESS_TOKEN" +# Use the default GitHub token if available. +# Skip this step if running an Enterprise instance. The default token there does not work for github.com. +elif [[ -n "${GITHUB_TOKEN:-}" && $GITHUB_SERVER_URL == "https://github.com" ]]; then + echo "::debug::Using the default GITHUB_TOKEN for github.com" + add_config "access-tokens = github.com=$GITHUB_TOKEN" +else + echo "::debug::Continuing without a GitHub access token" +fi +# Append extra nix configuration if provided +if [[ -n "${INPUT_EXTRA_NIX_CONFIG:-}" ]]; then + add_config "$INPUT_EXTRA_NIX_CONFIG" +fi +if [[ ! $INPUT_EXTRA_NIX_CONFIG =~ "experimental-features" ]]; then + add_config "experimental-features = nix-command flakes" +fi +# Always allow substituting from the cache, even if the derivation has `allowSubstitutes = false`. +# This is a CI optimisation to avoid having to download the inputs for already-cached derivations to rebuild trivial text files. +if [[ ! $INPUT_EXTRA_NIX_CONFIG =~ "always-allow-substitutes" ]]; then + add_config "always-allow-substitutes = true" +fi + +# Nix installer flags +installer_options=( + --no-channel-add + --darwin-use-unencrypted-nix-store-volume + --nix-extra-conf-file "$workdir/nix.conf" +) + +# only use the nix-daemon settings if on darwin (which get ignored) or systemd is supported +if [[ (! $INPUT_INSTALL_OPTIONS =~ "--no-daemon") && ($OSTYPE =~ darwin || -e /run/systemd/system) ]]; then + installer_options+=( + --daemon + --daemon-user-count "$(python3 -c 'import multiprocessing as mp; print(mp.cpu_count() * 2)')" + ) +else + # "fix" the following error when running nix* + # error: the group 'nixbld' specified in 'build-users-group' does not exist + add_config "build-users-group =" + sudo mkdir -p /etc/nix + sudo chmod 0755 /etc/nix + sudo cp "$workdir/nix.conf" /etc/nix/nix.conf +fi + +if [[ -n "${INPUT_INSTALL_OPTIONS:-}" ]]; then + IFS=' ' read -r -a extra_installer_options <<< "$INPUT_INSTALL_OPTIONS" + installer_options=("${extra_installer_options[@]}" "${installer_options[@]}") +fi + +echo "installer options: ${installer_options[*]}" + +# There is --retry-on-errors, but only newer curl versions support that +curl_retries=5 +while ! curl -sS -o "$workdir/install" -v --fail -L "${INPUT_INSTALL_URL:-https://releases.nixos.org/nix/nix-2.25.2/install}" +do + sleep 1 + ((curl_retries--)) + if [[ $curl_retries -le 0 ]]; then + echo "curl retries failed" >&2 + exit 1 + fi +done + +sh "$workdir/install" "${installer_options[@]}" + +# Set paths +echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" +# new path for nix 2.14 +echo "$HOME/.nix-profile/bin" >> "$GITHUB_PATH" + +if [[ -n "${INPUT_NIX_PATH:-}" ]]; then + echo "NIX_PATH=${INPUT_NIX_PATH}" >> "$GITHUB_ENV" +fi + +# Set temporary directory (if not already set) to fix https://github.com/cachix/matrixai-env-setup-action/issues/197 +if [[ -z "${TMPDIR:-}" ]]; then + echo "TMPDIR=${RUNNER_TEMP}" >> "$GITHUB_ENV" +fi + +if [[ -n "${INPUT_REGISTRY:-}" ]]; then + # Output the INPUT_REGISTRY variable contents to /etc/nix/registry.json + echo "$INPUT_REGISTRY" | sudo tee /etc/nix/registry.json > /dev/null +fi + +# Close the log message group which was opened above +echo "::endgroup::" diff --git a/.github/workflows/application-js-cloudflare-feature-closed.yml b/.github/workflows/application-js-cloudflare-feature-closed.yml index 771fbfd..a3ca582 100644 --- a/.github/workflows/application-js-cloudflare-feature-closed.yml +++ b/.github/workflows/application-js-cloudflare-feature-closed.yml @@ -3,6 +3,10 @@ name: "CI / Application JS Cloudflare Feature Closed" on: workflow_call: inputs: + environment: + description: 'Deployment Environment' + type: string + required: true appName: type: string required: true @@ -26,8 +30,6 @@ jobs: feature-closed-deployment-stop: name: "Feature Closed / Deployment Stop" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner concurrency: group: feature-closed-deployment-stop cancel-in-progress: false @@ -35,12 +37,13 @@ jobs: # This means the feature branch PR is closed if: startsWith(inputs.featureBranch, 'feature') permissions: - packages: read contents: read + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Stop Deployment env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} diff --git a/.github/workflows/application-js-cloudflare-feature.yml b/.github/workflows/application-js-cloudflare-feature.yml index 6f4669e..3cf4a3a 100644 --- a/.github/workflows/application-js-cloudflare-feature.yml +++ b/.github/workflows/application-js-cloudflare-feature.yml @@ -3,6 +3,10 @@ name: "CI / Application JS Cloudflare Feature" on: workflow_call: inputs: + environment: + description: 'Deployment Environment' + type: string + required: true ref: type: string default: master @@ -16,20 +20,22 @@ on: DEPLOY_SECRETS: required: true +env: + NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + jobs: # Lint the code feature-lint: name: "Feature / Lint" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run linting env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} @@ -42,16 +48,28 @@ jobs: feature-build: name: "Feature / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write + environment: ${{ inputs.environment }} steps: + - name: Checkout Actions + uses: actions/checkout@v4 + with: + repository: MatrixAI/.github + ref: ${{ inputs.ref }} + path: tmp/.github + - name: Parse Secrets + uses: ./tmp/.github/.github/actions/secrets-parse + with: + secrets: ${{ secrets.DEPLOY_SECRETS }} - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Setup Deploy Secrets + run: | + echo "${{ inputs.DEPLOY_SECRETS }}" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"' >> $GITHUB_ENV - name: Run build env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} @@ -70,11 +88,10 @@ jobs: name: "Feature / Deployment" runs-on: ubuntu-latest needs: feature-build - container: - image: ghcr.io/matrixai/github-runner concurrency: group: feature-deployment cancel-in-progress: false + environment: ${{ inputs.environment }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -89,17 +106,23 @@ jobs: - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - uses: actions/download-artifact@v4 with: name: public path: ./public + - name: Setup Deploy Secrets + run: | + echo "${{ inputs.DEPLOY_SECRETS }}" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"' >> $GITHUB_ENV - name: Run deployment env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + name: "feature/${{ github.ref_name }}" + url: "https://${{ github.ref_name }}.dev.zeta.house" run: | echo 'Perform service deployment for feature' nix develop .#ci --command bash -c $' npm run deploy -- \ --feature "$GITHUB_REF_NAME" \ --env "$GITHUB_REF_NAME" - ' \ No newline at end of file + ' diff --git a/.github/workflows/application-js-cloudflare-master.yml b/.github/workflows/application-js-cloudflare-master.yml index 8c07e3c..a993baf 100644 --- a/.github/workflows/application-js-cloudflare-master.yml +++ b/.github/workflows/application-js-cloudflare-master.yml @@ -3,6 +3,10 @@ name: "CI / Application JS Cloudflare Master" on: workflow_call: inputs: + environment: + description: 'Deployment Environment' + type: string + required: true ref: type: string default: master @@ -21,16 +25,28 @@ jobs: master-build: name: "Master / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write + environment: ${{ inputs.environment }} steps: + - name: Checkout Actions + uses: actions/checkout@v4 + with: + repository: MatrixAI/.github + ref: ${{ inputs.ref }} + path: tmp/.github + - name: Parse Secrets + uses: ./tmp/.github/.github/actions/secrets-parse + with: + secrets: ${{ secrets.DEPLOY_SECRETS }} - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Setup Deploy Secrets + run: | + echo "${{ inputs.DEPLOY_SECRETS }}" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"' >> $GITHUB_ENV - name: Run build env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} @@ -49,14 +65,12 @@ jobs: name: "Master / Deployment" runs-on: ubuntu-latest needs: master-build - container: - image: ghcr.io/matrixai/github-runner concurrency: group: master-deployment cancel-in-progress: false permissions: - packages: read contents: read + environment: ${{ inputs.environment }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -64,6 +78,7 @@ jobs: repository: MatrixAI/.github ref: ${{ inputs.ref }} path: tmp/.github + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Parse Secrets uses: ./tmp/.github/.github/actions/secrets-parse with: @@ -82,4 +97,4 @@ jobs: echo 'Perform service deployment for master' nix develop .#ci --command bash -c $' npm run deploy -- --env master - ' \ No newline at end of file + ' diff --git a/.github/workflows/application-js-cloudflare-staging.yml b/.github/workflows/application-js-cloudflare-staging.yml index 2019385..60cc507 100644 --- a/.github/workflows/application-js-cloudflare-staging.yml +++ b/.github/workflows/application-js-cloudflare-staging.yml @@ -3,6 +3,10 @@ name: "CI / Application JS Cloudflare Staging" on: workflow_call: inputs: + environment: + description: 'Deployment Environment' + type: string + required: true ref: type: string default: master @@ -31,15 +35,14 @@ jobs: staging-lint: name: "Staging / Lint" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run linting env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} @@ -53,21 +56,20 @@ jobs: name: "Staging / Merge Begin" runs-on: ubuntu-latest permissions: - packages: read contents: read pull-requests: write + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 - name: Create Pull Request from Staging to Master env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr create \ --head staging \ --base master \ --title "ci: merge staging to master" \ --body "This is an automatic PR generated by the CI/CD pipeline. This will be automatically fast-forward merged if successful." \ - --assignee "@me" \ --no-maintainer-edit || true printf "Pipeline Attempt on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ | gh pr comment staging \ @@ -78,16 +80,28 @@ jobs: staging-build: name: "Staging / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write + environment: ${{ inputs.environment }} steps: + - name: Checkout Actions + uses: actions/checkout@v4 + with: + repository: MatrixAI/.github + ref: ${{ inputs.ref }} + path: tmp/.github + - name: Parse Secrets + uses: ./tmp/.github/.github/actions/secrets-parse + with: + secrets: ${{ secrets.DEPLOY_SECRETS }} - uses: actions/checkout@v4 with: lfs: true + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Setup Deploy Secrets + run: | + echo "${{ inputs.DEPLOY_SECRETS }}" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"' >> $GITHUB_ENV - name: Run build env: NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} @@ -106,14 +120,12 @@ jobs: name: "Staging / Deployment" runs-on: ubuntu-latest needs: staging-build - container: - image: ghcr.io/matrixai/github-runner concurrency: group: staging-deployment cancel-in-progress: false permissions: - packages: read contents: read + environment: ${{ inputs.environment }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -121,6 +133,7 @@ jobs: repository: MatrixAI/.github ref: ${{ inputs.ref }} path: tmp/.github + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Parse Secrets uses: ./tmp/.github/.github/actions/secrets-parse with: @@ -153,9 +166,9 @@ jobs: group: staging-merge-finish cancel-in-progress: true permissions: - packages: read contents: write pull-requests: write + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 with: @@ -164,7 +177,7 @@ jobs: token: ${{ secrets.GH_TOKEN }} - name: Merge Pull Request from Staging to Master env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} @@ -176,4 +189,4 @@ jobs: --repo "$GITHUB_REPOSITORY" git checkout master git merge --ff-only "$GITHUB_SHA" - git push origin master \ No newline at end of file + git push origin master diff --git a/.github/workflows/application-js-feature.yml b/.github/workflows/application-js-feature.yml new file mode 100644 index 0000000..d69c4b1 --- /dev/null +++ b/.github/workflows/application-js-feature.yml @@ -0,0 +1,42 @@ +name: "CI / Application JS Feature" + +on: + workflow_call: + secrets: + NIXPKGS_PRIVATE_PAT: + required: true + +jobs: + # Lint the code + feature-lint: + name: "Feature / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + env: + NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Run a dry run + feature-dry: + name: "Feature / Dry Run" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Dry run + env: + NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + run: | + nix build .#default --dry-run + nix build .#docker --dry-run + diff --git a/.github/workflows/application-js-staging.yml b/.github/workflows/application-js-staging.yml new file mode 100644 index 0000000..5e9bdce --- /dev/null +++ b/.github/workflows/application-js-staging.yml @@ -0,0 +1,109 @@ +name: "CI / Application JS Staging" + +on: + workflow_call: + secrets: + NIXPKGS_PRIVATE_PAT: + required: true + GH_TOKEN: + required: true + GIT_AUTHOR_EMAIL: + required: true + GIT_AUTHOR_NAME: + required: true + GIT_COMMITTER_EMAIL: + required: true + GIT_COMMITTER_NAME: + required: true + +jobs: + # Lint the code + staging-lint: + name: "Staging / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + env: + NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Run a dry run + staging-dry: + name: "Staging / Dry Run" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Dry run + env: + NIX_CONFIG: access-tokens = github.com=${{ secrets.NIXPKGS_PRIVATE_PAT }} + run: | + nix build .#default --dry-run + nix build .#docker --dry-run + + # Create the merge PR + staging-merge-begin: + name: "Staging / Merge Begin" + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + - name: Create Pull Request from Staging to Master + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the CI/CD pipeline. This will be automatically fast-forward merged if successful." \ + --no-maintainer-edit || true + printf "Pipeline Attempt on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GITHUB_REPOSITORY" + + staging-merge-finish: + name: "Staging / Merge Finish" + needs: + - staging-lint + - staging-dry + - staging-merge-begin + runs-on: ubuntu-latest + concurrency: + group: staging-merge-finish + cancel-in-progress: true + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_TOKEN }} + - name: Merge Pull Request from Staging to Master + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} + GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} + GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} + GIT_COMMITTER_NAME: ${{ secrets.GIT_COMMITTER_NAME }} + run: | + printf "Pipeline Succeeded on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GITHUB_REPOSITORY" + git checkout master + git merge --ff-only "$GITHUB_SHA" + git push origin master diff --git a/.github/workflows/library-js-feature.yml b/.github/workflows/library-js-feature.yml index a3745f4..7a2e41a 100644 --- a/.github/workflows/library-js-feature.yml +++ b/.github/workflows/library-js-feature.yml @@ -8,13 +8,11 @@ jobs: feature-lint: name: "Feature / Lint" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run linting run: | nix develop .#ci --command bash -c $' @@ -25,14 +23,12 @@ jobs: feature-build: name: "Feature / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run build run: | nix develop .#ci --command bash -c $' @@ -48,15 +44,13 @@ jobs: feature-test: name: "Feature / Test" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write checks: write steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run tests run: | nix develop .#ci --command bash -c $' @@ -83,14 +77,12 @@ jobs: feature-bench: name: "Feature / Bench" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run bench run: | nix develop .#ci --command bash -c $' diff --git a/.github/workflows/library-js-staging.yml b/.github/workflows/library-js-staging.yml index 7407227..4e92368 100644 --- a/.github/workflows/library-js-staging.yml +++ b/.github/workflows/library-js-staging.yml @@ -19,13 +19,11 @@ jobs: staging-lint: name: "Staging / Lint" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run linting run: | nix develop .#ci --command bash -c $' @@ -37,21 +35,19 @@ jobs: name: "Staging / Merge Begin" runs-on: ubuntu-latest permissions: - packages: read contents: read pull-requests: write steps: - uses: actions/checkout@v4 - name: Create Pull Request from Staging to Master env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr create \ --head staging \ --base master \ --title "ci: merge staging to master" \ --body "This is an automatic PR generated by the CI/CD pipeline. This will be automatically fast-forward merged if successful." \ - --assignee "@me" \ --no-maintainer-edit || true printf "Pipeline Attempt on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ | gh pr comment staging \ @@ -62,14 +58,12 @@ jobs: staging-build: name: "Staging / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run build run: | nix develop .#ci --command bash -c $' @@ -88,10 +82,7 @@ jobs: needs: - staging-build runs-on: ${{ matrix.os }} - container: - image: ${{ matrix.platform == 'linux' && 'ghcr.io/matrixai/github-runner' || null }} permissions: - packages: read contents: read actions: write checks: write @@ -101,45 +92,59 @@ jobs: include: - platform: linux os: ubuntu-latest - env: {} - script: | - nix develop .#ci --command bash -c $' - npm test -- --ci --coverage - npm run bench --if-present - ' - platform: windows - os: windows-latest - env: {} - script: | - mkdir -Force "$CI_PROJECT_DIR/tmp" - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 - ./scripts/choco-install.ps1 - refreshenv - npm install --ignore-scripts - $env:Path = "$(npm root)\.bin;" + $env:Path - npm test -- --ci --coverage - npm run bench --if-present + os: windows-2022 - platform: macos os: macos-latest - env: {} - script: | - mkdir -p "$CI_PROJECT_DIR/tmp" - eval "$(brew shellenv)" - ./scripts/brew-install.sh - hash -r - npm install --ignore-scripts - export PATH="$(npm root)/.bin:$PATH" - npm test -- --ci --coverage - npm run bench --if-present steps: - uses: actions/checkout@v4 + - if: matrix.platform == 'linux' + uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - uses: actions/download-artifact@v4 with: name: dist path: ./dist - - name: Build - env: ${{ matrix.env }} - run: ${{ matrix.script }} + - name: Windows Bootstrap + if: matrix.platform == 'windows' + shell: pwsh + run: | + mkdir -Force "$CI_PROJECT_DIR/tmp" + ./scripts/choco-install.ps1 + where.exe node + where.exe npm + node -v + npm -v + - name: Build (Windows) + if: matrix.platform == 'windows' + shell: pwsh + run: | + npm install --ignore-scripts + $env:Path = "$(npm root)\.bin;" + $env:Path + where.exe node + where.exe npm + node -v + npm -v + npm test -- --ci --coverage + npm run bench --if-present + - name: Build (Linux) + if: matrix.platform == 'linux' + run: | + nix develop .#ci --command bash -c $' + npm test -- --ci --coverage + npm run bench --if-present + ' + - name: Build (macOS) + if: matrix.platform == 'macos' + shell: bash + run: | + mkdir -p "$CI_PROJECT_DIR/tmp" + eval "$(brew shellenv)" + ./scripts/brew-install.sh + hash -r + npm install --ignore-scripts + export PATH="$(npm root)/.bin:$PATH" + npm test -- --ci --coverage + npm run bench --if-present - name: Upload JUnit Report if: success() || failure() uses: actions/upload-artifact@v4 @@ -177,7 +182,6 @@ jobs: group: staging-merge-finish cancel-in-progress: true permissions: - packages: read contents: write pull-requests: write steps: @@ -187,7 +191,7 @@ jobs: token: ${{ secrets.GH_TOKEN }} - name: Merge Pull Request from Staging to Master env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} @@ -199,4 +203,4 @@ jobs: --repo "$GITHUB_REPOSITORY" git checkout master git merge --ff-only "$GITHUB_SHA" - git push origin master \ No newline at end of file + git push origin master diff --git a/.github/workflows/library-js-tag.yml b/.github/workflows/library-js-tag.yml index 04e23c2..d63f0ef 100644 --- a/.github/workflows/library-js-tag.yml +++ b/.github/workflows/library-js-tag.yml @@ -11,14 +11,12 @@ jobs: tag-build: name: "Tag / Build" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner permissions: - packages: read contents: read actions: write steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - name: Run build run: | nix develop .#ci --command bash -c $' @@ -34,19 +32,17 @@ jobs: tag-prerelease: name: "Tag / Pre-release" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner concurrency: group: tag-prerelease cancel-in-progress: false needs: - tag-build permissions: - packages: read contents: read if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-') steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - uses: actions/download-artifact@v4 with: name: dist @@ -68,19 +64,17 @@ jobs: tag-release: name: "Tag / Release" runs-on: ubuntu-latest - container: - image: ghcr.io/matrixai/github-runner concurrency: group: tag-release cancel-in-progress: false needs: - tag-build permissions: - packages: read contents: read if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') steps: - uses: actions/checkout@v4 + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master - uses: actions/download-artifact@v4 with: name: dist diff --git a/.github/workflows/native-library-js-feature.yml b/.github/workflows/native-library-js-feature.yml new file mode 100644 index 0000000..a3975ce --- /dev/null +++ b/.github/workflows/native-library-js-feature.yml @@ -0,0 +1,104 @@ +name: "CI / Library JS Feature" + +on: + workflow_call: + +jobs: + # Lint the code + feature-lint: + name: "Feature / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Build the dist + feature-build: + name: "Feature / Build" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run build + run: | + nix develop .#ci --command bash -c $' + npm run build --verbose + ' + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: dist + path: ./dist + + # Test the dist + feature-test: + name: "Feature / Test" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + checks: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run tests + run: | + nix develop .#ci --command bash -c $' + npm run test -- --ci --coverage + ' + - name: Upload JUnit report + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: junit-report + path: tmp/junit/junit.xml + - name: Publish JUnit Report + uses: mikepenz/action-junit-report@v5 + with: + report_paths: tmp/junit/junit.xml + - name: Upload Cobertura report + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: tmp/coverage/cobertura-coverage.xml + + # Bench the dist + feature-bench: + name: "Feature / Bench" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run bench + run: | + nix develop .#ci --command bash -c $' + npm run bench --if-present + ' + - name: Upload Bench + uses: actions/upload-artifact@v4 + with: + name: metrics-report + path: ./benches/results/metrics.txt + if-no-files-found: ignore diff --git a/.github/workflows/native-library-js-staging.yml b/.github/workflows/native-library-js-staging.yml new file mode 100644 index 0000000..2ff19d1 --- /dev/null +++ b/.github/workflows/native-library-js-staging.yml @@ -0,0 +1,219 @@ +name: "CI / Native Library JS Staging" + +on: + workflow_call: + secrets: + GH_TOKEN: + required: true + GIT_AUTHOR_EMAIL: + required: true + GIT_AUTHOR_NAME: + required: true + GIT_COMMITTER_EMAIL: + required: true + GIT_COMMITTER_NAME: + required: true + +jobs: + # Lint the code + staging-lint: + name: "Staging / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Create the merge PR + staging-merge-begin: + name: "Staging / Merge Begin" + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Create Pull Request from Staging to Master + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the CI/CD pipeline. This will be automatically fast-forward merged if successful." \ + --no-maintainer-edit || true + printf "Pipeline Attempt on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GITHUB_REPOSITORY" + + # Build the distribution - JS is platform-agnostic + staging-build: + name: "Staging / Build" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run build + run: | + nix develop .#ci --command bash -c $' + npm run build --ignore-scripts --verbose + ' + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: dist + path: ./dist + + # Build on every platform + # This re-uses the built `./dist`, and run tests and benches + staging-platforms: + name: "Staging / Platforms" + needs: + - staging-build + runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: write + checks: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: ubuntu-latest + - platform: windows + os: windows-2022 + - platform: macos + os: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - if: matrix.platform == 'linux' + uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + name: dist + path: ./dist + - name: Windows Bootstrap + if: matrix.platform == 'windows' + shell: pwsh + run: | + mkdir -Force "$CI_PROJECT_DIR/tmp" + ./scripts/choco-install.ps1 + where.exe node + where.exe npm + node -v + npm -v + - name: Build (Windows) + if: matrix.platform == 'windows' + shell: pwsh + run: | + npm install --ignore-scripts + $env:Path = "$(npm root)\.bin;" + $env:Path + where.exe node + where.exe npm + node -v + npm -v + npm run prebuild --verbose + npm test -- --ci --coverage + npm run bench --if-present + - name: Build (Linux) + if: matrix.platform == 'linux' + run: | + nix develop .#ci --command bash -c $' + npm run prebuild --verbose + npm test -- --ci --coverage + npm run bench --if-present + ' + - name: Build (macOS) + if: matrix.platform == 'macos' + shell: bash + run: | + mkdir -p "$CI_PROJECT_DIR/tmp" + eval "$(brew shellenv)" + ./scripts/brew-install.sh + export PYTHON=$(brew --prefix python@3.10)/bin/python3.10 + hash -r + npm install --ignore-scripts + export PATH="$(npm root)/.bin:$PATH" + npm run prebuild --verbose + npm test -- --ci --coverage + npm run bench --if-present + - name: Upload JUnit Report + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: junit-report-${{ matrix.platform }} + path: ./tmp/junit/junit.xml + - name: Publish JUnit Report + uses: mikepenz/action-junit-report@v5 + with: + check_name: JUnit Test Report - ${{matrix.platform}} + report_paths: ./tmp/junit/junit.xml + - name: Upload Cobertura report + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-report-${{ matrix.platform }} + path: ./tmp/coverage/cobertura-coverage.xml + - name: Upload Metrics Report + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: metrics-report-${{ matrix.platform }} + path: ./benches/results/metrics.txt + if-no-files-found: ignore + + staging-merge-finish: + name: "Staging / Merge Finish" + needs: + - staging-lint + - staging-merge-begin + - staging-build + - staging-platforms + runs-on: ubuntu-latest + concurrency: + group: staging-merge-finish + cancel-in-progress: true + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + token: ${{ secrets.GH_TOKEN }} + - name: Merge Pull Request from Staging to Master + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} + GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} + GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} + GIT_COMMITTER_NAME: ${{ secrets.GIT_COMMITTER_NAME }} + run: | + printf "Pipeline Succeeded on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GITHUB_REPOSITORY" + git checkout master + git merge --ff-only "$GITHUB_SHA" + git push origin master diff --git a/.github/workflows/native-library-js-tag-gyp.yml b/.github/workflows/native-library-js-tag-gyp.yml new file mode 100644 index 0000000..5025739 --- /dev/null +++ b/.github/workflows/native-library-js-tag-gyp.yml @@ -0,0 +1,243 @@ +name: "CI / Library JS Tag" + +on: + workflow_call: + secrets: + NPM_TOKEN: + required: true + +jobs: + # Lint the code + tag-lint: + name: "Tag / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Build the distribution - JS is platform-agnostic + tag-build: + name: "Tag / Build" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + needs: tag-lint + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run build + run: | + nix develop .#ci --command bash -c $' + npm run build --verbose + ' + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: dist + path: ./dist + + tag-platforms: + name: "Tag / Platforms" + needs: + - tag-build + runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: write + checks: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: ubuntu-latest + - platform: windows + os: windows-2022 + - platform: macos + os: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - if: matrix.platform == 'linux' + uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + name: dist + path: ./dist + - name: Windows Bootstrap + if: matrix.platform == 'windows' + shell: pwsh + run: | + mkdir -Force "$CI_PROJECT_DIR/tmp" + ./scripts/choco-install.ps1 + where.exe node + where.exe npm + node -v + npm -v + - name: Build (Windows) + if: matrix.platform == 'windows' + shell: pwsh + env: + npm_config_arch: "x64" + RUST_BACKTRACE: "1" + LIBCLANG_PATH: "C:\\Program Files\\LLVM\\bin" + run: | + npm install --ignore-scripts + $env:Path = "$(npm root)\.bin;" + $env:Path + where.exe node + where.exe npm + node -v + npm -v + npm run prebuild --verbose -- --production + npm test -- --ci --coverage + npm run bench + - name: Build (Linux) + if: matrix.platform == 'linux' + env: + npm_config_arch: "x64" + RUST_BACKTRACE: "1" + run: | + nix develop .#ci --command bash -c $' + npm run prebuild --verbose -- --production + npm test -- --ci --coverage + npm run bench + ' + - name: Build (macOS) + if: matrix.platform == 'macos' + shell: bash + run: | + eval "$(brew shellenv)" + ./scripts/brew-install.sh + export PYTHON=$(brew --prefix python@3.10)/bin/python3.10 + hash -r + npm install --ignore-scripts + export PATH="$(npm root)/.bin:$PATH" + npm run prebuild --verbose -- --production + npm test -- --ci --coverage + npm run bench + - uses: actions/upload-artifact@v4 + with: + name: prebuild-${{ matrix.platform }} + path: ./prebuilds + + + # Publish the prerelease + tag-prerelease: + name: "Tag / Pre-release" + runs-on: ubuntu-latest + concurrency: + group: tag-prerelease + cancel-in-progress: false + needs: + - tag-platforms + permissions: + contents: write + if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-') + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + pattern: prebuild* + path: prebuilds + merge-multiple: true + - name: Publishing library prerelease + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + nix develop .#ci --command bash -c $' + npm publish --tag prerelease --access public + ' + for d in prebuilds/*; do + tar \ + --create \ + --verbose \ + --file="prebuilds/$(basename $d).tar" \ + --directory=prebuilds \ + "$(basename $d)" + done + nix develop .#ci --command bash -c $' + gh release \ + create "$GITHUB_REF_NAME" \ + prebuilds/*.tar \ + --title "$GITHUB_REF_NAME-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --prerelease \ + --target staging \ + --repo "$GITHUB_REPOSITORY" + ' + - name: Remove `.npmrc` + if: success() || failure() + run: | + rm -f ./.npmrc + + # Publish the release + tag-release: + name: "Tag / Release" + runs-on: ubuntu-latest + concurrency: + group: tag-release + cancel-in-progress: false + needs: + - tag-platforms + permissions: + contents: write + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + pattern: prebuild* + path: prebuilds + merge-multiple: true + - name: Publishing library release + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + nix develop .#ci --command bash -c $' + npm publish --access public + ' + for d in prebuilds/*; do + tar \ + --create \ + --verbose \ + --file="prebuilds/$(basename $d).tar" \ + --directory=prebuilds \ + "$(basename $d)" + done + nix develop .#ci --command bash -c $' + gh release \ + create "$GITHUB_REF_NAME" \ + prebuilds/*.tar \ + --title "$GITHUB_REF_NAME-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --target master \ + --repo "$GITHUB_REPOSITORY" + ' + - name: Remove `.npmrc` + if: success() || failure() + run: | + rm -f ./.npmrc diff --git a/.github/workflows/native-library-js-tag.yml b/.github/workflows/native-library-js-tag.yml new file mode 100644 index 0000000..141da76 --- /dev/null +++ b/.github/workflows/native-library-js-tag.yml @@ -0,0 +1,258 @@ +name: "CI / Library JS Tag" + +on: + workflow_call: + secrets: + NPM_TOKEN: + required: true + +jobs: + # Lint the code + tag-lint: + name: "Tag / Lint" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run linting + run: | + nix develop .#ci --command bash -c $' + npm run lint + ' + + # Build the distribution - JS is platform-agnostic + tag-build: + name: "Tag / Build" + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + needs: tag-lint + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - name: Run build + run: | + nix develop .#ci --command bash -c $' + npm run build --verbose + ' + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: dist + path: ./dist + + tag-platforms: + name: "Tag / Platforms" + needs: + - tag-build + runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: write + checks: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: ubuntu-latest + - platform: windows + os: windows-2022 + - platform: macos + os: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - if: matrix.platform == 'linux' + uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + name: dist + path: ./dist + - name: Windows Bootstrap + if: matrix.platform == 'windows' + shell: pwsh + run: | + mkdir -Force "$CI_PROJECT_DIR/tmp" + ./scripts/choco-install.ps1 + where.exe node + where.exe npm + node -v + npm -v + - name: Build (Windows) + if: matrix.platform == 'windows' + shell: pwsh + env: + npm_config_arch: "x64" + RUST_BACKTRACE: "1" + LIBCLANG_PATH: "C:\\Program Files\\LLVM\\bin" + run: | + npm install --ignore-scripts + $env:Path = "$(npm root)\.bin;" + $env:Path + where.exe node + where.exe npm + node -v + npm -v + npm run prebuild --verbose -- --production + npm test -- --ci --coverage + npm run bench + - name: Build (Linux) + if: matrix.platform == 'linux' + env: + npm_config_arch: "x64" + RUST_BACKTRACE: "1" + run: | + nix develop .#ci --command bash -c $' + npm run prebuild --verbose -- --production + npm test -- --ci --coverage + npm run bench + ' + - name: Build (macOS) + if: matrix.platform == 'macos' + shell: bash + run: | + eval "$(brew shellenv)" + ./scripts/brew-install.sh + export PYTHON=$(brew --prefix python@3.10)/bin/python3.10 + hash -r + npm install --ignore-scripts + export PATH="$(npm root)/.bin:$PATH" + export PATH="$HOME/.cargo/bin:$PATH" + echo "Prebuilding for darwin-x64" + npm run prebuild --verbose -- --arch x64 --production + echo "Prebuilding for darwin-arm64" + npm run prebuild --verbose -- --arch arm64 --production + echo "Creating universal binary" + for f in prebuild/*-darwin-arm64.node; do + prefix=$(basename "$f" | sed -E 's/-darwin-arm64\.node$//') + lipo -create \ + -output "prebuild/${prefix}-darwin-x64+arm64.node" \ + "prebuild/${prefix}-darwin-arm64.node" \ + "prebuild/${prefix}-darwin-x64.node" + done + rm -rf node_modules/@matrixai/*-* + echo "Running tests" + npm test -- --ci --coverage + echo "Running benchmarks" + npm run bench + - uses: actions/upload-artifact@v4 + with: + name: prebuild-${{ matrix.platform }} + path: ./prebuild + + + # Publish the prerelease + tag-prerelease: + name: "Tag / Pre-release" + runs-on: ubuntu-latest + concurrency: + group: tag-prerelease + cancel-in-progress: false + needs: + - tag-platforms + permissions: + contents: write + if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-') + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + pattern: prebuild* + path: prebuild + merge-multiple: true + - name: Publishing library prerelease + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + nix develop .#ci --command bash -c $' + npm publish --tag prerelease --access public + ' + for d in prebuild/*; do + tar \ + --create \ + --verbose \ + --file="prebuild/$(basename $d).tar" \ + --directory=prebuild \ + "$(basename $d)" + done + nix develop .#ci --command bash -c $' + gh release \ + create "$GITHUB_REF_NAME" \ + prebuild/*.tar \ + --title "$GITHUB_REF_NAME-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --prerelease \ + --target staging \ + --repo "$GITHUB_REPOSITORY" + ' + - name: Remove `.npmrc` + if: success() || failure() + run: | + rm -f ./.npmrc + + # Publish the release + tag-release: + name: "Tag / Release" + runs-on: ubuntu-latest + concurrency: + group: tag-release + cancel-in-progress: false + needs: + - tag-platforms + permissions: + contents: write + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: MatrixAI/.github/.github/actions/matrixai-env-setup@master + - uses: actions/download-artifact@v4 + with: + pattern: prebuild* + path: prebuild + merge-multiple: true + - name: Publishing library release + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + nix develop .#ci --command bash -c $' + npm publish --access public + ' + for d in prebuild/*; do + tar \ + --create \ + --verbose \ + --file="prebuild/$(basename $d).tar" \ + --directory=prebuild \ + "$(basename $d)" + done + nix develop .#ci --command bash -c $' + gh release \ + create "$GITHUB_REF_NAME" \ + prebuild/*.tar \ + --title "$GITHUB_REF_NAME-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --target master \ + --repo "$GITHUB_REPOSITORY" + ' + - name: Remove `.npmrc` + if: success() || failure() + run: | + rm -f ./.npmrc diff --git a/.gitlab/issue_templates/bug_report.md b/.gitlab/issue_templates/bug_report.md deleted file mode 100644 index 5499b1b..0000000 --- a/.gitlab/issue_templates/bug_report.md +++ /dev/null @@ -1,29 +0,0 @@ -### Describe the bug - - -### To Reproduce - -1. ... -2. ... -3. ... - -### Expected behavior - - -### Screenshots - - -### Platform (please complete the following information) - - Device: [e.g. iPhone6] - - OS: [e.g. iOS] - - Version [e.g. 22] - -### Additional context - - -### Notify maintainers - - -/label ~bug \ No newline at end of file diff --git a/.gitlab/issue_templates/design.md b/.gitlab/issue_templates/design.md deleted file mode 100644 index 60e595f..0000000 --- a/.gitlab/issue_templates/design.md +++ /dev/null @@ -1,22 +0,0 @@ -### Requirements of this design - -1. ... -2. ... -3. ... - -### Additional context - - -### Specification - -1. ... -2. ... -3. ... - -### Sub-Issues & Sub-PRs created - -1. ... -2. ... -3. ... - -/label ~design ~enhancement \ No newline at end of file diff --git a/.gitlab/issue_templates/development.md b/.gitlab/issue_templates/development.md deleted file mode 100644 index b42937d..0000000 --- a/.gitlab/issue_templates/development.md +++ /dev/null @@ -1,13 +0,0 @@ -### Specification - - -### Additional context - - -### Tasks - -1. ... -2. ... -3. ... - -/label ~development \ No newline at end of file diff --git a/.gitlab/issue_templates/feature_request.md b/.gitlab/issue_templates/feature_request.md deleted file mode 100644 index 5d2a31c..0000000 --- a/.gitlab/issue_templates/feature_request.md +++ /dev/null @@ -1,13 +0,0 @@ -### Is your feature request related to a problem? Please describe. - - -### Describe the solution you'd like - - -### Describe alternatives you've considered - - -### Additional context - - -/label ~enhancement \ No newline at end of file diff --git a/.gitlab/issue_templates/procedure.md b/.gitlab/issue_templates/procedure.md deleted file mode 100644 index 03b228b..0000000 --- a/.gitlab/issue_templates/procedure.md +++ /dev/null @@ -1,11 +0,0 @@ -### Tasks - -- [ ] 1. ... -- [ ] 2. ... -- [ ] 3. ... - -/label ~procedure \ No newline at end of file diff --git a/.gitlab/issue_templates/research.md b/.gitlab/issue_templates/research.md deleted file mode 100644 index 5954057..0000000 --- a/.gitlab/issue_templates/research.md +++ /dev/null @@ -1,23 +0,0 @@ -### What is your research hypothesis/question? - - -### Review existing ideas, literature and prior work - -- [ ] 1. ... -- [ ] 2. ... -- [ ] 3. ... - -### Research conclusion - - -### Sub-Issues & Sub-PRs created - -1. ... -2. ... -3. ... - -/label ~research \ No newline at end of file diff --git a/.gitlab/merge_request_templates/development.md b/.gitlab/merge_request_templates/development.md deleted file mode 100644 index a8d7d77..0000000 --- a/.gitlab/merge_request_templates/development.md +++ /dev/null @@ -1,26 +0,0 @@ -### Description - - -### Issues Fixed - -* Fixes #... - -### Tasks - -- [ ] 1. ... -- [ ] 2. ... -- [ ] 3. ... - -### Final checklist - - -* [ ] Domain specific tests -* [ ] Full tests -* [ ] Updated inline-comment documentation -* [ ] Lint fixed -* [ ] Squash and rebased -* [ ] Sanity check the final build diff --git a/README.md b/README.md index d53e87e..bce300d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Templates +## Issues, Pull Requests, Merge Requests + This repository stories issue, pull request and merge request templates for Matrix AI GitHub templates are stored in `.github`. This repository must be placed under https://github.com/MatrixAI/.github. The `.github` repository name is essential for GitHub to recognise it as a special organisation-template repository. @@ -15,14 +17,32 @@ Ensure that this repository is mirrored from GitLab to GitHub. This is where we centralized re-usable workflows for GitHub actions. Workflows are grouped together into different kinds of projects. +### Windows runners and Node.js + +On GitHub-hosted Windows runners it is common to have multiple Node versions on `PATH` (for example Node 22 may be preinstalled). If a workflow calls `refreshenv` / `Update-SessionEnvironment` after selecting Node, it can rebuild `PATH` and cause `npm.cmd` to come from a different Node installation than `node.exe`. + +To avoid the Node/npm mismatch: + +* Windows jobs are pinned to `windows-2022`. +* Windows execution is split into two steps: + 1. **Bootstrap** runs repo-local `./scripts/choco-install.ps1` and prints: + `where.exe node`, `where.exe npm`, `node -v`, `npm -v`, `npm exec --yes node -v`. + 2. **Build/Test** runs npm commands in a separate step so `$GITHUB_PATH` updates from the bootstrap step are applied deterministically. +* Reusable workflows in this repository do not use `actions/setup-node` for Windows; they delegate Node selection to the downstream repo’s `./scripts/choco-install.ps1`. + * library-js - TS/JS projects that produce libraries as NPM packages. - feature - for feature branches - staging - for staging branches - tag - for tag branches -* library-js-native - TS/JS projets that produce libraries using native code as NPM packages. +* library-js-native - TS/JS projects that produce libraries using native code as NPM packages. - feature - staging - tag +* application-js-cloudflare - TS/JS projects that produce Cloudflare applications + - feature + - staging + - master + - feature-closed To use them, for example in a library-js project. You create 3 caller workflows in `/.github/workflows`: @@ -44,7 +64,6 @@ in `/.github/workflows`: jobs: use-library-js-feature: permissions: - packages: read contents: read actions: write checks: write @@ -67,7 +86,6 @@ in `/.github/workflows`: jobs: use-library-js-staging: permissions: - packages: read contents: read actions: write checks: write @@ -88,9 +106,8 @@ in `/.github/workflows`: jobs: use-library-js-tag: permissions: - packages: read contents: read actions: write uses: MatrixAI/.github/.github/workflows/library-js-tag.yml@master secrets: inherit - ``` \ No newline at end of file + ```