From 935472fc836ac666d0209933bb778708bbefd8c5 Mon Sep 17 00:00:00 2001 From: Stephen Shen Date: Tue, 2 Dec 2025 12:47:33 -0500 Subject: [PATCH 1/6] Add workflows --- .github/workflows/create-release.yml | 110 ++++++ .../monitor-upstream-and-analyze.yml | 367 ++++++++++++++++++ .github/workflows/monitor-upstream.yml | 300 ++++++++++++++ .github/workflows/run-analysis.yml | 28 ++ .github/workflows/run-filter.yml | 38 ++ .github/workflows/set-cache-sha.yml | 49 +++ 6 files changed, 892 insertions(+) create mode 100644 .github/workflows/create-release.yml create mode 100644 .github/workflows/monitor-upstream-and-analyze.yml create mode 100644 .github/workflows/monitor-upstream.yml create mode 100644 .github/workflows/run-analysis.yml create mode 100644 .github/workflows/run-filter.yml create mode 100644 .github/workflows/set-cache-sha.yml diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000..40d92702 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,110 @@ +name: Create Release for Analysis Artifacts + +on: + workflow_dispatch: + inputs: + prefix: + description: "Artifact name prefix" + required: true + type: string + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + + steps: + - name: Create timestamp + id: ts + run: | + ts="$(date -u +%Y%m%dT%H%M%SZ)" + echo "ts=$ts" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + id: create-release + run: | + tag="analysis-${{ steps.ts.outputs.ts }}" + name="Continuous Analysis Release (${{ steps.ts.outputs.ts }})" + + echo "Creating release: $name" + + response=$(curl -s -X POST \ + -H "Authorization: token $GH_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"tag_name\": \"$tag\", \"name\": \"$name\", \"draft\": false}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases") + + upload_url=$(echo "$response" | jq -r '.upload_url' | sed 's/{?name,label}//') + echo "upload_url=$upload_url" >> $GITHUB_OUTPUT + + - name: Fetch all artifact metadata + id: fetch + run: | + echo "Fetching all artifacts..." + + page=1 + all="[]" + while true; do + response=$(curl -s -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100&page=$page") + + artifacts=$(echo "$response" | jq -c '.artifacts[]?') + if [[ -z "$artifacts" ]]; then break; fi + + while IFS= read -r art; do + all=$(echo "$all" | jq --argjson a "$art" '. + [$a]') + done <<< "$artifacts" + + count=$(echo "$response" | jq '.artifacts | length') + (( count < 100 )) && break + + ((page++)) + done + + echo "$all" > all_artifacts.json + echo "Saved metadata for all artifacts." + + - name: Upload matching artifacts to release + run: | + prefix="${{ inputs.prefix }}" + upload_url="${{ steps.create-release.outputs.upload_url }}" + + echo "Looking for artifacts starting with: $prefix" + echo "" + + matches=$(jq -c --arg p "$prefix" '.[] | select(.name | startswith($p))' all_artifacts.json) + + if [[ "$(echo "$matches" | wc -l)" -eq 0 ]]; then + echo "❌ No artifacts found starting with: $prefix" + exit 1 + fi + + echo "$matches" | while IFS= read -r art; do + name=$(echo "$art" | jq -r '.name') + id=$(echo "$art" | jq -r '.id') + zip="${name}.zip" + + echo "▶ Downloading artifact: $name (ID $id)" + + curl -L -s \ + -H "Authorization: token $GH_TOKEN" \ + -o "$zip" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" + + echo "⬆ Uploading $zip to release..." + + curl -s -X POST \ + -H "Authorization: token $GH_TOKEN" \ + -H "Content-Type: application/zip" \ + --data-binary @"$zip" \ + "${upload_url}?name=${zip}" + + echo "✓ Uploaded: $zip" + echo "" + done + + echo "🎉 Release upload completed successfully!" diff --git a/.github/workflows/monitor-upstream-and-analyze.yml b/.github/workflows/monitor-upstream-and-analyze.yml new file mode 100644 index 00000000..f11e8885 --- /dev/null +++ b/.github/workflows/monitor-upstream-and-analyze.yml @@ -0,0 +1,367 @@ +name: Monitor Upstream and Run Analysis + +on: + schedule: + - cron: "0 */6 * * *" + workflow_dispatch: + inputs: + number_of_commits: + description: "Historical mode: analyze N previous commits (0 = continuous mode)" + required: false + type: number + default: 0 + skip_commits: + description: "Skip commit pattern: process every (N+1)th commit (0 = process all)" + required: false + type: number + default: 0 + +permissions: + actions: write + contents: write + issues: write + +jobs: + monitor-upstream: + runs-on: ubuntu-latest + + env: + # NEED TO BE CONFIGURED EACH PROJECT + UPSTREAM_REPO: "blaxel-ai/sdk-python" + BRANCH: "main" + RUNNER_DISPATCH_TIMEOUT: 7200 # 2 hours + FILTER_DISPATCH_TIMEOUT: 1800 # 30 minutes + MAX_CONCURRENT: 8 + + steps: + # -------------------------------------------------------------------- + # STEP 1 — SYNC FORK WITH UPSTREAM + # -------------------------------------------------------------------- + - name: Checkout fork + uses: actions/checkout@v4 + with: + token: ${{ secrets.ORG_WIDE_TOKEN }} + path: forked-repo + fetch-depth: 0 + + - name: Sync fork with upstream + run: | + set -euo pipefail + cd forked-repo + + echo "Syncing ${BRANCH} with upstream ${UPSTREAM_REPO}..." + + git config --global --add safe.directory "$PWD" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + git fetch --prune origin + git remote add upstream "https://github.com/${UPSTREAM_REPO}.git" || true + git fetch --prune upstream --tags + + git checkout "${BRANCH}" + + echo "Rebasing local branch onto upstream/${BRANCH}..." + if ! git rebase -X theirs --rebase-merges "upstream/${BRANCH}"; then + echo "Rebase failed -- aborting." + git rebase --abort || true + exit 1 + fi + + if ! git diff --quiet "origin/${BRANCH}..HEAD"; then + git push --force-with-lease origin "${BRANCH}" + echo "Rebase changes pushed to origin." + else + echo "No diverged changes; nothing to push." + fi + + cd .. + + - name: Cleanup forked repo + run: rm -rf forked-repo + + # -------------------------------------------------------------------- + # STEP 2 — LOAD LAST-SEEN SHA (or compute from history) + # -------------------------------------------------------------------- + - name: Restore analysis cache folder + id: cache-folder + uses: actions/cache/restore@v4 + with: + path: .continuous-analysis-cache + key: continuous-analysis-cache-${{ github.repository }}- + + - name: Ensure cache folder exists + run: mkdir -p .continuous-analysis-cache + + - name: Determine last-seen SHA + id: last-sha + run: | + NUMBER_OF_COMMITS="${{ inputs.number_of_commits }}" + + # --------------------------- + # Historical mode + # --------------------------- + if [[ "$NUMBER_OF_COMMITS" -gt 0 ]]; then + echo "Historical mode: computing boundary SHA for ${NUMBER_OF_COMMITS} commits." + + git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + cd upstream-repo + git checkout "${BRANCH}" + git rev-list --first-parent "${BRANCH}" > ../linear_commits.txt + cd .. + rm -rf upstream-repo + + TOTAL=$(wc -l < linear_commits.txt) + echo "Total commits in first-parent history: $TOTAL" + + if [[ "$NUMBER_OF_COMMITS" -ge "$TOTAL" ]]; then + echo "ERROR: number_of_commits $NUMBER_OF_COMMITS exceeds available history ($TOTAL)." >&2 + exit 1 + fi + + LAST_SHA=$(sed -n "$((NUMBER_OF_COMMITS + 1))p" linear_commits.txt) + echo "Historical boundary last_sha = $LAST_SHA" + echo "last_sha=$LAST_SHA" >> $GITHUB_OUTPUT + exit 0 + fi + + # --------------------------- + # Continuous mode + # --------------------------- + CACHE_FILE=".continuous-analysis-cache/last_sha.txt" + + if [[ -f "$CACHE_FILE" ]]; then + LAST_SHA=$(cat "$CACHE_FILE") + echo "Loaded last-seen SHA from cache: $LAST_SHA" + else + LAST_SHA="" + echo "No last-seen SHA found; this is the first run." + fi + + echo "last_sha=$LAST_SHA" >> $GITHUB_OUTPUT + + # -------------------------------------------------------------------- + # STEP 3 — FETCH COMMIT HISTORY & COMPUTE NEW COMMITS + # -------------------------------------------------------------------- + - name: Fetch first-parent commit history + id: check-commits + run: | + echo "Fetching upstream first-parent history..." + + git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + cd upstream-repo + git checkout "${BRANCH}" + git rev-list --first-parent "${BRANCH}" > ../all_commits.txt + cd .. + rm -rf upstream-repo + + echo "Total first-parent commits: $(wc -l < all_commits.txt)" + + LAST_SEEN="${{ steps.last-sha.outputs.last_sha }}" + SKIP_COMMITS="${{ inputs.skip_commits }}" + + [[ -z "$SKIP_COMMITS" || "$SKIP_COMMITS" -lt 0 ]] && SKIP_COMMITS=0 + + # First-time run + if [[ -z "$LAST_SEEN" ]]; then + head -n 1 all_commits.txt > new_commits.txt + echo "Initial run: analyzing latest commit only: $(cat new_commits.txt)" + echo "has_new_commits=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Extract new commits above LAST_SEEN + awk -v sha="$LAST_SEEN" '$0 ~ sha {exit} {print}' all_commits.txt > temp_new_commits.txt + + if [[ ! -s temp_new_commits.txt ]]; then + echo "No new commits since last analysis." + touch new_commits.txt + echo "has_new_commits=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # Apply skip pattern + if [[ "$SKIP_COMMITS" -eq 0 ]]; then + mv temp_new_commits.txt new_commits.txt + else + awk "NR % ($SKIP_COMMITS + 1) == 1" temp_new_commits.txt > new_commits.txt + rm temp_new_commits.txt + fi + + echo "has_new_commits=true" >> $GITHUB_OUTPUT + echo "New commits to analyze (total: $(wc -l < new_commits.txt)):" + cat new_commits.txt + + # -------------------------------------------------------------------- + # STEP 4 — GENERATE DISPATCH ID (ONLY FOR HISTORICAL MODE) + # -------------------------------------------------------------------- + - name: Generate dispatch ID + id: dispatch-id + run: | + if [[ "${{ inputs.number_of_commits }}" -gt 0 ]]; then + dispatch_id="$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM" + echo "Historical run dispatch ID: $dispatch_id" + echo "dispatch_id=$dispatch_id" >> $GITHUB_OUTPUT + else + echo "dispatch_id=" >> $GITHUB_OUTPUT + fi + + # -------------------------------------------------------------------- + # STEP 5 — TRIGGER ANALYSIS WORKFLOWS (PARALLEL) + # -------------------------------------------------------------------- + - name: Run analysis workflows + if: steps.check-commits.outputs.has_new_commits == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mapfile -t commits < new_commits.txt + dispatch_id="${{ steps.dispatch-id.outputs.dispatch_id }}" + repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) + + MAX_CONCURRENT=${{ env.MAX_CONCURRENT }} + total_commits=${#commits[@]} + + echo "Launching analysis for ${total_commits} commits..." + + for ((batch_start=0; batch_start total_commits)) && batch_end=$total_commits + + echo "Processing batch $((batch_start/MAX_CONCURRENT + 1))..." + + declare -a dispatched_commits=() + declare -a artifact_names=() + + for ((i=batch_start; i=0; i--)); do + reversed+=("${commits[$i]}") + done + + for i in "${!reversed[@]}"; do + current="${reversed[$i]}" + previous="" + ((i > 0)) && previous="${reversed[$i-1]}" + + echo "Filtering: $current (prev: $previous)" + + if [[ "${{ inputs.number_of_commits }}" -gt 0 ]]; then + artifact="continuous-analysis-history-filtered-results-${dispatch_id}-${repo_name}-${current}-${{ inputs.skip_commits }}" + else + artifact="continuous-analysis-filtered-results-${repo_name}-${current}" + fi + + gh workflow run run-filter.yml \ + --repo "${GITHUB_REPOSITORY}" \ + --ref "${BRANCH}" \ + --field current_commit="$current" \ + --field previous_commit="$previous" \ + --field dispatch_id="$dispatch_id" \ + --field skip_commits_pattern="${{ inputs.skip_commits }}" + + end_time=$(( $(date +%s) + FILTER_DISPATCH_TIMEOUT )) + created=false + + while ! $created && [[ $(date +%s) -lt $end_time ]]; do + names=$(curl -s -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100" \ + | jq -r '.artifacts[].name') + if echo "$names" | grep -q "^${artifact}"; then + echo " ✓ Filter artifact: ${artifact}" + created=true + break + fi + sleep 30 + done + + if ! $created; then + echo "ERROR: Timeout waiting for filter artifact: ${artifact}" + exit 1 + fi + done + + # -------------------------------------------------------------------- + # STEP 7 — UPDATE SHA CACHE (Continuous mode only) + # -------------------------------------------------------------------- + - name: Update last-seen SHA cache + if: steps.check-commits.outputs.has_new_commits == 'true' && inputs.number_of_commits == 0 + run: | + NEWEST=$(head -n 1 new_commits.txt) + echo "$NEWEST" > .continuous-analysis-cache/last_sha.txt + echo "Updated cache: last_sha = $NEWEST" + + - name: Generate timestamp + if: steps.check-commits.outputs.has_new_commits == 'true' && inputs.number_of_commits == 0 + id: timestamp + run: | + ts=$(date +'%Y%m%d-%H%M') + echo "ts=$ts" >> "$GITHUB_OUTPUT" + + - name: Save updated cache + if: steps.check-commits.outputs.has_new_commits == 'true' && inputs.number_of_commits == 0 + uses: actions/cache/save@v4 + with: + path: .continuous-analysis-cache + key: continuous-analysis-cache-${{ github.repository }}-${{ steps.timestamp.outputs.ts }} diff --git a/.github/workflows/monitor-upstream.yml b/.github/workflows/monitor-upstream.yml new file mode 100644 index 00000000..d1ecd215 --- /dev/null +++ b/.github/workflows/monitor-upstream.yml @@ -0,0 +1,300 @@ +name: Monitor Upstream Repository + +on: + workflow_dispatch: + inputs: + number_of_commits: + description: "Historical mode: analyze N previous commits (0 = continuous mode)" + required: true + type: number + skip_commits: + description: "Skip commit pattern: process every (N+1)th commit (0 = process all)" + required: false + type: number + default: 0 + +permissions: + actions: write + contents: write + issues: write + +jobs: + monitor-upstream: + runs-on: ubuntu-latest + + env: + # NEED TO BE CONFIGURED EACH PROJECT + UPSTREAM_REPO: "blaxel-ai/sdk-python" + BRANCH: "main" + RUNNER_DISPATCH_TIMEOUT: 7200 # 2 hours + MAX_CONCURRENT: 9 + + steps: + # -------------------------------------------------------------------- + # STEP 1 — COMPUTE LAST-SEEN SHA BASED ON NUMBER OF COMMITS + # -------------------------------------------------------------------- + - name: Determine last-seen SHA + id: last-sha + run: | + # Set error handling + set -euo pipefail + + # Get the number of commits to analyze + NUMBER_OF_COMMITS="${{ inputs.number_of_commits }}" + echo "Historical mode: computing boundary SHA for ${NUMBER_OF_COMMITS} commits." + + # Clone the upstream repository + git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + cd upstream-repo + git checkout "${BRANCH}" + git rev-list --first-parent "${BRANCH}" > ../linear_commits.txt + cd .. + rm -rf upstream-repo + + # Get the total number of commits in the first-parent history + TOTAL=$(wc -l < linear_commits.txt) + echo "Total commits in first-parent history: $TOTAL" + + # Check if the number of commits to analyze is greater than 0 + if [[ "$NUMBER_OF_COMMITS" -le 0 ]]; then + echo "ERROR: number_of_commits $NUMBER_OF_COMMITS is less than or equal to 0." >&2 + exit 1 + fi + + # Check if the number of commits to analyze exceeds the total number of commits in the first-parent history + if [[ "$NUMBER_OF_COMMITS" -ge "$TOTAL" ]]; then + echo "ERROR: number_of_commits $NUMBER_OF_COMMITS exceeds available history ($TOTAL)." >&2 + exit 1 + fi + + # Extract the last seen SHA based on the number of commits to analyze + LAST_SHA=$(sed -n "$((NUMBER_OF_COMMITS + 1))p" linear_commits.txt) + echo "Historical boundary last_sha = $LAST_SHA" + echo "last_sha=$LAST_SHA" >> $GITHUB_OUTPUT + + # -------------------------------------------------------------------- + # STEP 2 — FETCH COMMIT HISTORY & COMPUTE NEW COMMITS + # -------------------------------------------------------------------- + - name: Fetch first-parent commit history + id: check-commits + run: | + echo "Fetching upstream first-parent history..." + + git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + cd upstream-repo + git checkout "${BRANCH}" + git rev-list --first-parent "${BRANCH}" > ../all_commits.txt + cd .. + rm -rf upstream-repo + + echo "Total first-parent commits: $(wc -l < all_commits.txt)" + + LAST_SEEN="${{ steps.last-sha.outputs.last_sha }}" + SKIP_COMMITS="${{ inputs.skip_commits }}" + + [[ -z "$SKIP_COMMITS" || "$SKIP_COMMITS" -lt 0 ]] && SKIP_COMMITS=0 + + # Extract new commits above LAST_SEEN + awk -v sha="$LAST_SEEN" '$0 ~ sha {exit} {print}' all_commits.txt > temp_new_commits.txt + + # Apply skip pattern if skip_commits is provided + if [[ "$SKIP_COMMITS" -eq 0 ]]; then + mv temp_new_commits.txt new_commits.txt + else + awk "NR % ($SKIP_COMMITS + 1) == 1" temp_new_commits.txt > new_commits.txt + rm temp_new_commits.txt + fi + + # Print the new commits to be analyzed + echo "New commits to be analyzed (total: $(wc -l < new_commits.txt)):" + cat new_commits.txt + + # -------------------------------------------------------------------- + # STEP 3 — GENERATE DISPATCH ID + # -------------------------------------------------------------------- + - name: Generate dispatch ID + id: dispatch-id + run: | + dispatch_id="$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM" + echo "Dispatch ID: $dispatch_id" + echo "dispatch_id=$dispatch_id" >> $GITHUB_OUTPUT + + # -------------------------------------------------------------------- + # STEP 4 — RUN ANALYSIS (PARALLEL) AND COLLECT ARTIFACTS + # -------------------------------------------------------------------- + - name: Run analysis workflows (parallel) and collect artifacts + id: run-analysis + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + mapfile -t commits < new_commits.txt + dispatch_id="${{ steps.dispatch-id.outputs.dispatch_id }}" + repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) + MAX_CONCURRENT=${{ env.MAX_CONCURRENT }} + total_commits=${#commits[@]} + + echo "Launching analysis for ${total_commits} commits..." + echo "" > all_expected_artifacts.txt # all expected artifacts saved here + + # ---------------------------------------- + # DISPATCH ANALYSIS WORKFLOWS IN BATCHES + # ---------------------------------------- + for ((batch_start=0; batch_start total_commits)) && batch_end=$total_commits + + echo "Processing batch $((batch_start/MAX_CONCURRENT + 1))..." + + declare -a dispatched_commits=() + declare -a artifact_names=() + + # Dispatch each workflow in this batch + for ((i=batch_start; i> all_expected_artifacts.txt + + gh workflow run run-analysis.yml \ + --repo "${GITHUB_REPOSITORY}" \ + --ref "${BRANCH}" \ + --field commit="$commit" \ + --field dispatch_id="$dispatch_id" + + dispatched_commits+=("$commit") + artifact_names+=("$artifact_name") + done + + # ----------------------------- + # WAIT FOR THIS BATCH TO FINISH + # ----------------------------- + echo "Waiting for batch artifacts..." + + end_time=$(( $(date +%s) + RUNNER_DISPATCH_TIMEOUT )) + declare -a done=() + + while [[ ${#done[@]} -lt ${#dispatched_commits[@]} && $(date +%s) -lt $end_time ]]; do + artifact_list=$(curl -s -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100" \ + | jq -r '.artifacts[].name') + + for idx in "${!artifact_names[@]}"; do + if [[ " ${done[@]} " =~ " ${artifact_names[$idx]} " ]]; then continue; fi + if echo "$artifact_list" | grep -q "^${artifact_names[$idx]}$"; then + echo " ✓ Artifact found: ${artifact_names[$idx]}" + done+=("${artifact_names[$idx]}") + fi + done + + [[ ${#done[@]} -lt ${#artifact_names[@]} ]] && sleep 60 + done + + if [[ ${#done[@]} -lt ${#artifact_names[@]} ]]; then + echo "ERROR: Timeout while waiting for batch artifacts." + exit 1 + fi + + echo "Batch completed." + done + + echo "All analysis workflows completed." + + # -------------------------------------------------------------------- + # STEP 5 — CREATE ONE RELEASE AND UPLOAD ALL ARTIFACTS + # -------------------------------------------------------------------- + - name: Create release and upload artifacts + id: make-release + env: + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + run: | + set -euo pipefail + + # Helper function to fetch all artifacts across all pages + fetch_all_artifacts() { + local page=1 + local all_artifacts_array="[]" + while true; do + local response=$(curl -s -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100&page=${page}") + local page_artifacts=$(echo "$response" | jq -c '.artifacts[]') + if [[ -z "$page_artifacts" ]]; then + break + fi + # Merge this page's artifacts into the array + while IFS= read -r artifact; do + all_artifacts_array=$(echo "$all_artifacts_array" | jq --argjson art "$artifact" '. + [$art]') + done <<< "$page_artifacts" + local per_page=$(echo "$response" | jq -r '.artifacts | length') + if [[ $per_page -lt 100 ]]; then + break + fi + ((page++)) + done + echo "$all_artifacts_array" + } + + # Release naming (one release per workflow run) + run_timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + release_tag="analysis-${run_timestamp}" + release_name="Continuous Analysis Run ${run_timestamp}" + + echo "Creating release: $release_name" + + # Create release + api_response=$(curl -s -X POST \ + -H "Authorization: token $GH_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"tag_name\": \"${release_tag}\", \"name\": \"${release_name}\", \"draft\": false, \"prerelease\": false}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases") + + upload_url=$(echo "$api_response" | jq -r '.upload_url' | sed 's/{?name,label}//') + + echo "Upload URL: $upload_url" + + echo "Fetching all artifacts (this may take a moment for large runs)..." + all_artifacts_data=$(fetch_all_artifacts) + + echo "Uploading artifacts..." + + # For each expected artifact, download it & upload it + while IFS= read -r artifact_name; do + echo " Processing artifact: $artifact_name" + + # Query artifact metadata from the fetched data + artifact_info=$(echo "$all_artifacts_data" | jq -c --arg NAME "$artifact_name" '[.[] | select(.name == $NAME)][0]') + + if [[ -z "$artifact_info" || "$artifact_info" == "null" ]]; then + echo " WARNING: Artifact not found: $artifact_name" + continue + fi + + artifact_id=$(echo "$artifact_info" | jq -r '.id') + zip_name="${artifact_name}.zip" + + echo " Downloading artifact ID $artifact_id --> $zip_name" + + # Download ZIP + curl -L -s \ + -H "Authorization: token $GH_TOKEN" \ + -o "$zip_name" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" + + echo " Uploading $zip_name to release..." + + # Upload ZIP to release + curl -s -X POST \ + -H "Authorization: token $GH_TOKEN" \ + -H "Content-Type: application/zip" \ + --data-binary @"$zip_name" \ + "${upload_url}?name=${zip_name}" + + echo " ✓ Uploaded: $zip_name" + + done < all_expected_artifacts.txt + + echo "Release completed successfully." diff --git a/.github/workflows/run-analysis.yml b/.github/workflows/run-analysis.yml new file mode 100644 index 00000000..beba3690 --- /dev/null +++ b/.github/workflows/run-analysis.yml @@ -0,0 +1,28 @@ +name: Trigger Continuous Analysis + +on: + workflow_dispatch: + inputs: + commit: + description: 'Single commit SHA to test' + required: true + type: string + dispatch_id: + description: "Unique id from dispatcher for history runnings" + required: false + type: string + +permissions: + actions: read + contents: write + issues: write + +jobs: + analyze-single-commit: + uses: ContinuousAnalysis/continuous-analysis/.github/workflows/auto-runner.yml@main + with: + project: ${{ github.repository }} + commit: ${{ inputs.commit }} + dispatch_id: ${{ inputs.dispatch_id }} + secrets: + ORG_WIDE_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} diff --git a/.github/workflows/run-filter.yml b/.github/workflows/run-filter.yml new file mode 100644 index 00000000..ba2a56b5 --- /dev/null +++ b/.github/workflows/run-filter.yml @@ -0,0 +1,38 @@ +name: Trigger Continuous Analysis Violation Filter + +on: + workflow_dispatch: + inputs: + current_commit: + description: 'Current commit SHA to test' + required: true + type: string + previous_commit: + description: 'Previous commit SHA to test' + required: false + type: string + dispatch_id: + description: "Unique id from dispatcher for history runnings" + required: false + type: string + skip_commits_pattern: + description: "Number of commits to skip between processing (skip x commits pattern)" + required: false + type: number + +permissions: + actions: read + contents: write + issues: write + +jobs: + analyze-single-commit: + uses: ContinuousAnalysis/continuous-analysis/.github/workflows/auto-filter.yml@main + with: + project: ${{ github.repository }} + current_commit: ${{ inputs.current_commit }} + previous_commit: ${{ inputs.previous_commit }} + dispatch_id: ${{ inputs.dispatch_id }} + skip_commits_pattern: ${{ inputs.skip_commits_pattern }} + secrets: + ORG_WIDE_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} diff --git a/.github/workflows/set-cache-sha.yml b/.github/workflows/set-cache-sha.yml new file mode 100644 index 00000000..aee3ec81 --- /dev/null +++ b/.github/workflows/set-cache-sha.yml @@ -0,0 +1,49 @@ +name: Set Cache SHA + +on: + workflow_dispatch: + inputs: + commit_sha: + description: "Commit SHA to set in cache" + required: true + type: string + +permissions: + contents: write + +jobs: + set-cache-sha: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate and set commit SHA + run: | + COMMIT_SHA="${{ inputs.commit_sha }}" + + # Validate format + if [[ ! "$COMMIT_SHA" =~ ^[a-f0-9]{7,40}$ ]]; then + echo "Invalid commit SHA format" + exit 1 + fi + + # Create cache directory and set SHA + mkdir -p .continuous-analysis-cache + echo "$COMMIT_SHA" > .continuous-analysis-cache/last_sha.txt + + echo "Set cache SHA: $COMMIT_SHA" + + - name: Generate timestamp + id: timestamp + run: | + ts=$(date +'%Y%m%d-%H%M') + echo "ts=$ts" >> "$GITHUB_OUTPUT" + echo "Generated timestamp: $ts" + + - name: Save updated SHA cache with timestamp + uses: actions/cache/save@v4 + with: + path: .continuous-analysis-cache + key: continuous-analysis-cache-${{ github.repository }}-${{ steps.timestamp.outputs.ts }} From 2852c27199d83f0506a48d856687c8ec701735b6 Mon Sep 17 00:00:00 2001 From: Stephen Shen Date: Sun, 28 Dec 2025 14:24:02 -0500 Subject: [PATCH 2/6] Improve workflows --- .../monitor-upstream-and-analyze.yml | 104 +++++++++++++----- 1 file changed, 79 insertions(+), 25 deletions(-) diff --git a/.github/workflows/monitor-upstream-and-analyze.yml b/.github/workflows/monitor-upstream-and-analyze.yml index f11e8885..7f785a49 100644 --- a/.github/workflows/monitor-upstream-and-analyze.yml +++ b/.github/workflows/monitor-upstream-and-analyze.yml @@ -104,10 +104,11 @@ jobs: if [[ "$NUMBER_OF_COMMITS" -gt 0 ]]; then echo "Historical mode: computing boundary SHA for ${NUMBER_OF_COMMITS} commits." - git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + # Clone the upstream repository + git clone "https://github.com/${UPSTREAM_REPO}.git" upstream-repo cd upstream-repo git checkout "${BRANCH}" - git rev-list --first-parent "${BRANCH}" > ../linear_commits.txt + git log --no-merges --name-status | grep 'py\|^commit' | grep -B1 'py$' | grep ^commit | cut -d ' ' -f 2 > ../linear_commits.txt cd .. rm -rf upstream-repo @@ -143,19 +144,19 @@ jobs: # -------------------------------------------------------------------- # STEP 3 — FETCH COMMIT HISTORY & COMPUTE NEW COMMITS # -------------------------------------------------------------------- - - name: Fetch first-parent commit history + - name: Fetch commit history with python files changed and no merge commits id: check-commits run: | - echo "Fetching upstream first-parent history..." + echo "Fetching upstream commit history with python files changed and no merge commits..." - git clone --depth=100000 "https://github.com/${UPSTREAM_REPO}.git" upstream-repo + git clone "https://github.com/${UPSTREAM_REPO}.git" upstream-repo cd upstream-repo git checkout "${BRANCH}" - git rev-list --first-parent "${BRANCH}" > ../all_commits.txt + git log --no-merges --name-status | grep 'py\|^commit' | grep -B1 'py$' | grep ^commit | cut -d ' ' -f 2 > ../all_commits.txt cd .. rm -rf upstream-repo - echo "Total first-parent commits: $(wc -l < all_commits.txt)" + echo "Total commits with python files changed and no merge commits: $(wc -l < all_commits.txt)" LAST_SEEN="${{ steps.last-sha.outputs.last_sha }}" SKIP_COMMITS="${{ inputs.skip_commits }}" @@ -221,27 +222,75 @@ jobs: MAX_CONCURRENT=${{ env.MAX_CONCURRENT }} total_commits=${#commits[@]} - echo "Launching analysis for ${total_commits} commits..." + echo "Checking ${total_commits} commits for existing artifacts..." - for ((batch_start=0; batch_start total_commits)) && batch_end=$total_commits + ((batch_end > total_to_process)) && batch_end=$total_to_process echo "Processing batch $((batch_start/MAX_CONCURRENT + 1))..." declare -a dispatched_commits=() - declare -a artifact_names=() + declare -a dispatched_artifacts=() for ((i=batch_start; i 0)) && previous="${reversed[$i-1]}" + if [[ $i -eq 0 ]]; then + previous="$LAST_SEEN" + else + previous="${reversed[$i-1]}" + fi echo "Filtering: $current (prev: $previous)" if [[ "${{ inputs.number_of_commits }}" -gt 0 ]]; then artifact="continuous-analysis-history-filtered-results-${dispatch_id}-${repo_name}-${current}-${{ inputs.skip_commits }}" else - artifact="continuous-analysis-filtered-results-${repo_name}-${current}" + artifact="continuous-analysis-future-filtered-results-${repo_name}-${current}" fi gh workflow run run-filter.yml \ From 1d101f77bf2f39bb44e85bd247565a6734f3d321 Mon Sep 17 00:00:00 2001 From: Zhuohang Shen <18962118885@163.com> Date: Sun, 28 Dec 2025 14:43:32 -0500 Subject: [PATCH 3/6] Reduce max concurrent number --- .github/workflows/monitor-upstream-and-analyze.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/monitor-upstream-and-analyze.yml b/.github/workflows/monitor-upstream-and-analyze.yml index 7f785a49..3eddf306 100644 --- a/.github/workflows/monitor-upstream-and-analyze.yml +++ b/.github/workflows/monitor-upstream-and-analyze.yml @@ -31,7 +31,7 @@ jobs: BRANCH: "main" RUNNER_DISPATCH_TIMEOUT: 7200 # 2 hours FILTER_DISPATCH_TIMEOUT: 1800 # 30 minutes - MAX_CONCURRENT: 8 + MAX_CONCURRENT: steps: # -------------------------------------------------------------------- From 2d554a2a52b5f5adaa53776dd4f23ab156bb5c93 Mon Sep 17 00:00:00 2001 From: Stephen Shen Date: Sat, 21 Feb 2026 20:37:17 -0500 Subject: [PATCH 4/6] chore: update GitHub Actions workflows --- .../monitor-open-prs-and-analyze.yml | 368 ++++++++++++++++++ .../monitor-upstream-and-analyze.yml | 8 +- .github/workflows/monitor-upstream.yml | 3 +- .github/workflows/run-analysis.yml | 9 +- .github/workflows/run-filter.yml | 9 +- 5 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/monitor-open-prs-and-analyze.yml diff --git a/.github/workflows/monitor-open-prs-and-analyze.yml b/.github/workflows/monitor-open-prs-and-analyze.yml new file mode 100644 index 00000000..04d6ae5b --- /dev/null +++ b/.github/workflows/monitor-open-prs-and-analyze.yml @@ -0,0 +1,368 @@ +name: Monitor Open PRs and Run Analysis + +on: + schedule: + - cron: "30 */6 * * *" + workflow_dispatch: + +permissions: + actions: write + contents: write + pull-requests: read + +# NEED TO BE CONFIGURED EACH PROJECT +env: + UPSTREAM_REPO: "blaxel-ai/sdk-python" + BRANCH: "main" + RUNNER_DISPATCH_TIMEOUT: 7200 # 2 hours + FILTER_DISPATCH_TIMEOUT: 1800 # 30 minutes + MAX_CONCURRENT: 2 + +jobs: + monitor-open-prs: + runs-on: ubuntu-latest + + steps: + # -------------------------------------------------------------------- + # STEP 1 — FETCH OPEN PRs FROM UPSTREAM VIA GITHUB API + # -------------------------------------------------------------------- + - name: Fetch open PRs and merge bases from upstream + id: fetch-prs + env: + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + run: | + set -euo pipefail + + echo "Fetching open PRs from upstream ${UPSTREAM_REPO}..." + + # Read all open PRs from the upstream repository + prs_json="[]" + page=1 + per_page=100 + while true; do + batch=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${UPSTREAM_REPO}/pulls?state=open&per_page=${per_page}&page=${page}") + + # Check if the response is a valid array + if ! echo "$batch" | jq -e 'type == "array"' >/dev/null 2>&1; then + echo "API error or non-array response: $(echo "$batch" | jq -c . 2>/dev/null || echo "$batch")" + break + fi + + # Get the number of PRs in the batch and add them to the prs_json array + count=$(echo "$batch" | jq 'length') + [[ "$count" -eq 0 ]] && break + prs_json=$(echo "$prs_json" "$batch" | jq -s 'add') + [[ "$count" -lt "$per_page" ]] && break + page=$((page + 1)) + done + + # Get the total number of PRs in the prs_json array + pr_count=$(echo "$prs_json" | jq 'length') + echo "Found $pr_count open PR(s) in upstream." + + # If there are no PRs, exit + if [[ "$pr_count" -eq 0 ]]; then + echo '[]' > prs_to_analyze.json + echo "pr_count=0" >> $GITHUB_OUTPUT + echo "No PRs to analyze in upstream." + exit 0 + fi + + # Get the repository name + repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) + results="[]" + base_ref="${BRANCH}" + + # Iterate over the PRs and add them to the prs_to_analyze.json array + for idx in $(seq 0 $((pr_count - 1))); do + head_repo=$(echo "$prs_json" | jq -r ".[$idx].head.repo.full_name") + + # Check if the PR is from the upstream repository + if [[ "$head_repo" != "$UPSTREAM_REPO" ]]; then + + # If the PR is not from the upstream repository, skip it + pr_num=$(echo "$prs_json" | jq -r ".[$idx].number") + echo "Skipping PR #${pr_num} (head from fork: ${head_repo})" + continue + fi + + # Get the head SHA and PR number + head_sha=$(echo "$prs_json" | jq -r ".[$idx].head.sha") + pr_num=$(echo "$prs_json" | jq -r ".[$idx].number") + head_ref=$(echo "$prs_json" | jq -r ".[$idx].head.ref") + + # Get the merge base SHA + compare=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${UPSTREAM_REPO}/compare/${base_ref}...${head_sha}") + merge_base_sha=$(echo "$compare" | jq -r '.merge_base_commit.sha // empty') + + # If the merge base SHA is not found, skip the PR + if [[ -z "$merge_base_sha" ]] || [[ "$merge_base_sha" == "null" ]]; then + echo "Skipping PR #${pr_num}: could not get merge base" + continue + fi + + # Get the files changed in the PR to make sure there are Python files changed + files=$(echo "$compare" | jq -r '.files[]?.filename // empty') + has_py=false + if [[ -n "$files" ]]; then + while IFS= read -r f; do + [[ -z "$f" ]] && continue + if [[ "$f" == *.py ]]; then has_py=true; break; fi + done <<< "$files" + fi + + # If there are no Python files changed, skip the PR + if ! $has_py; then + echo "Skipping PR #${pr_num}: no Python files changed" + continue + fi + + # One dispatch_id per PR, reused in analysis and filter steps + dispatch_id="pr-${pr_num}-$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM" + echo "PR #${pr_num} dispatch_id=${dispatch_id} head=${head_sha} merge_base=${merge_base_sha} (branch ${head_ref})" + entry=$(jq -n \ + --arg pr "$pr_num" \ + --arg head "$head_sha" \ + --arg base "$merge_base_sha" \ + --arg ref "$head_ref" \ + --arg did "$dispatch_id" \ + '{pr_number: $pr, head_sha: $head, merge_base_sha: $base, head_ref: $ref, dispatch_id: $did}') + results=$(echo "$results" | jq --argjson e "$entry" '. + [$e]') + done + + # Save the prs_to_analyze.json array to a file + echo "$results" > prs_to_analyze.json + count=$(echo "$results" | jq 'length') + echo "pr_count=$count" >> $GITHUB_OUTPUT + echo "PRs to analyze: $count" + + # -------------------------------------------------------------------- + # STEP 2 — FETCH PR HEADS FROM UPSTREAM INTO CURRENT REPO + # -------------------------------------------------------------------- + - name: Fetch upstream PR refs and push into current repo + if: steps.fetch-prs.outputs.pr_count != '0' # Only run if there are PRs to analyze + env: + PUSH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + run: | + set -euo pipefail + + # Clone the current repository into a new directory + git clone --no-tags "https://x-access-token:${PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" push-repo + cd push-repo + + # Configure the git user + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Add the upstream repository as a remote + git remote add upstream "https://github.com/${UPSTREAM_REPO}.git" || true + git fetch upstream --no-tags + + # Iterate over the PRs and fetch the head SHA into the current repository + for idx in $(seq 0 $(($(jq 'length' ../prs_to_analyze.json) - 1))); do + pr_num=$(jq -r ".[$idx].pr_number" ../prs_to_analyze.json) + head_sha=$(jq -r ".[$idx].head_sha" ../prs_to_analyze.json) + echo "Fetching upstream PR #${pr_num} (${head_sha}) into current repo..." + git fetch upstream "refs/pull/${pr_num}/head:pr-${pr_num}" || true + git push origin "pr-${pr_num}" || true + done + + # Go back to the root directory and remove the temporary repository + cd .. + rm -rf push-repo + echo "PR heads are now in current repo." + + # -------------------------------------------------------------------- + # STEP 3 — TRIGGER ANALYSIS WORKFLOWS + # -------------------------------------------------------------------- + - name: Run analysis workflows for PRs + if: steps.fetch-prs.outputs.pr_count != '0' # Only run if there are PRs to analyze + env: + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + BRANCH: ${{ env.BRANCH }} + run: | + set -euo pipefail + + # Get the repository name, runner dispatch timeout and max concurrent + repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) + RUNNER_DISPATCH_TIMEOUT=${{ env.RUNNER_DISPATCH_TIMEOUT }} + MAX_CONCURRENT=${{ env.MAX_CONCURRENT }} + + # Get the total number of PRs to analyze + total_prs=$(jq 'length' prs_to_analyze.json) + echo "Processing ${total_prs} PR(s) in batches of ${MAX_CONCURRENT}..." + + # Iterate over the PRs in batches of max concurrent + for ((batch_start=0; batch_start total_prs)) && batch_end=$total_prs + batch_size=$((batch_end - batch_start)) + echo "Batch: PRs $((batch_start+1))–${batch_end} (${batch_size} PRs concurrently)" + + # Initialize array to store the batch artifacts + declare -a batch_artifacts=() + + # Iterate over the PRs in the batch (reuse dispatch_id from Step 1) + for ((idx=batch_start; idx total_prs)) && batch_end=$total_prs + batch_size=$((batch_end - batch_start)) + echo "Filter batch: PRs $((batch_start+1))–${batch_end} (${batch_size} PRs)" + + # --- Phase 1: dispatch and wait for BASE filter (reuse dispatch_id from Step 1) --- + declare -a batch_base_artifacts=() + for ((idx=batch_start; idx> $GITHUB_OUTPUT echo "dispatch_id=$dispatch_id" >> $GITHUB_OUTPUT else + echo "dispatch_type=" >> $GITHUB_OUTPUT echo "dispatch_id=" >> $GITHUB_OUTPUT fi @@ -216,6 +218,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mapfile -t commits < new_commits.txt + dispatch_type="${{ steps.dispatch-id.outputs.dispatch_type }}" dispatch_id="${{ steps.dispatch-id.outputs.dispatch_id }}" repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) @@ -295,6 +298,7 @@ jobs: --repo "${GITHUB_REPOSITORY}" \ --ref "${BRANCH}" \ --field commit="$commit" \ + --field dispatch_type="$dispatch_type" \ --field dispatch_id="$dispatch_id" dispatched_commits+=("$commit") @@ -339,6 +343,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mapfile -t commits < new_commits.txt + dispatch_type="${{ steps.dispatch-id.outputs.dispatch_type }}" dispatch_id="${{ steps.dispatch-id.outputs.dispatch_id }}" repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) @@ -372,6 +377,7 @@ jobs: --ref "${BRANCH}" \ --field current_commit="$current" \ --field previous_commit="$previous" \ + --field dispatch_type="$dispatch_type" \ --field dispatch_id="$dispatch_id" \ --field skip_commits_pattern="${{ inputs.skip_commits }}" diff --git a/.github/workflows/monitor-upstream.yml b/.github/workflows/monitor-upstream.yml index d1ecd215..deb6ef2f 100644 --- a/.github/workflows/monitor-upstream.yml +++ b/.github/workflows/monitor-upstream.yml @@ -27,7 +27,7 @@ jobs: UPSTREAM_REPO: "blaxel-ai/sdk-python" BRANCH: "main" RUNNER_DISPATCH_TIMEOUT: 7200 # 2 hours - MAX_CONCURRENT: 9 + MAX_CONCURRENT: 2 steps: # -------------------------------------------------------------------- @@ -164,6 +164,7 @@ jobs: --repo "${GITHUB_REPOSITORY}" \ --ref "${BRANCH}" \ --field commit="$commit" \ + --field dispatch_type="history" \ --field dispatch_id="$dispatch_id" dispatched_commits+=("$commit") diff --git a/.github/workflows/run-analysis.yml b/.github/workflows/run-analysis.yml index beba3690..9fe6b4d1 100644 --- a/.github/workflows/run-analysis.yml +++ b/.github/workflows/run-analysis.yml @@ -7,10 +7,16 @@ on: description: 'Single commit SHA to test' required: true type: string + dispatch_type: + description: "Type of dispatch: history or prs (empty for continuous mode)" + required: false + type: string + default: "" dispatch_id: - description: "Unique id from dispatcher for history runnings" + description: "Unique id from dispatcher for history / PR runnings" required: false type: string + default: "" permissions: actions: read @@ -23,6 +29,7 @@ jobs: with: project: ${{ github.repository }} commit: ${{ inputs.commit }} + dispatch_type: ${{ inputs.dispatch_type }} dispatch_id: ${{ inputs.dispatch_id }} secrets: ORG_WIDE_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} diff --git a/.github/workflows/run-filter.yml b/.github/workflows/run-filter.yml index ba2a56b5..962f2860 100644 --- a/.github/workflows/run-filter.yml +++ b/.github/workflows/run-filter.yml @@ -11,10 +11,16 @@ on: description: 'Previous commit SHA to test' required: false type: string + dispatch_type: + description: "Type of dispatch: history or prs (empty for continuous mode)" + required: false + type: string + default: "" dispatch_id: - description: "Unique id from dispatcher for history runnings" + description: "Unique id from dispatcher for history / PR runnings" required: false type: string + default: "" skip_commits_pattern: description: "Number of commits to skip between processing (skip x commits pattern)" required: false @@ -32,6 +38,7 @@ jobs: project: ${{ github.repository }} current_commit: ${{ inputs.current_commit }} previous_commit: ${{ inputs.previous_commit }} + dispatch_type: ${{ inputs.dispatch_type }} dispatch_id: ${{ inputs.dispatch_id }} skip_commits_pattern: ${{ inputs.skip_commits_pattern }} secrets: From 7075a38895d17c4a9e21c778bcaf650054bb8b99 Mon Sep 17 00:00:00 2001 From: Stephen Shen Date: Sat, 21 Feb 2026 21:28:16 -0500 Subject: [PATCH 5/6] chore: update GitHub Actions workflows schedule --- .github/workflows/monitor-open-prs-and-analyze.yml | 2 +- .github/workflows/monitor-upstream-and-analyze.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/monitor-open-prs-and-analyze.yml b/.github/workflows/monitor-open-prs-and-analyze.yml index 04d6ae5b..7e56217c 100644 --- a/.github/workflows/monitor-open-prs-and-analyze.yml +++ b/.github/workflows/monitor-open-prs-and-analyze.yml @@ -2,7 +2,7 @@ name: Monitor Open PRs and Run Analysis on: schedule: - - cron: "30 */6 * * *" + - cron: "0 6 * * *" workflow_dispatch: permissions: diff --git a/.github/workflows/monitor-upstream-and-analyze.yml b/.github/workflows/monitor-upstream-and-analyze.yml index 44bc0842..e405fd29 100644 --- a/.github/workflows/monitor-upstream-and-analyze.yml +++ b/.github/workflows/monitor-upstream-and-analyze.yml @@ -2,7 +2,7 @@ name: Monitor Upstream and Run Analysis on: schedule: - - cron: "0 */6 * * *" + - cron: "0 18 * * *" workflow_dispatch: inputs: number_of_commits: From f18c56f78ef6cd72139052222a2a01a83441eeaa Mon Sep 17 00:00:00 2001 From: Stephen Shen Date: Sun, 22 Feb 2026 11:32:43 -0500 Subject: [PATCH 6/6] chore: update GitHub Actions workflows schedule --- .../monitor-open-prs-and-analyze.yml | 103 +++++++++++++----- .../monitor-upstream-and-analyze.yml | 2 +- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/.github/workflows/monitor-open-prs-and-analyze.yml b/.github/workflows/monitor-open-prs-and-analyze.yml index 7e56217c..e0255210 100644 --- a/.github/workflows/monitor-open-prs-and-analyze.yml +++ b/.github/workflows/monitor-open-prs-and-analyze.yml @@ -2,7 +2,7 @@ name: Monitor Open PRs and Run Analysis on: schedule: - - cron: "0 6 * * *" + - cron: "0 18 * * *" workflow_dispatch: permissions: @@ -78,14 +78,14 @@ jobs: for idx in $(seq 0 $((pr_count - 1))); do head_repo=$(echo "$prs_json" | jq -r ".[$idx].head.repo.full_name") - # Check if the PR is from the upstream repository - if [[ "$head_repo" != "$UPSTREAM_REPO" ]]; then + # # Check if the PR is from the upstream repository + # if [[ "$head_repo" != "$UPSTREAM_REPO" ]]; then - # If the PR is not from the upstream repository, skip it - pr_num=$(echo "$prs_json" | jq -r ".[$idx].number") - echo "Skipping PR #${pr_num} (head from fork: ${head_repo})" - continue - fi + # # If the PR is not from the upstream repository, skip it + # pr_num=$(echo "$prs_json" | jq -r ".[$idx].number") + # echo "Skipping PR #${pr_num} (head from fork: ${head_repo})" + # continue + # fi # Get the head SHA and PR number head_sha=$(echo "$prs_json" | jq -r ".[$idx].head.sha") @@ -144,12 +144,12 @@ jobs: - name: Fetch upstream PR refs and push into current repo if: steps.fetch-prs.outputs.pr_count != '0' # Only run if there are PRs to analyze env: - PUSH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} run: | set -euo pipefail # Clone the current repository into a new directory - git clone --no-tags "https://x-access-token:${PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" push-repo + git clone --no-tags "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" push-repo cd push-repo # Configure the git user @@ -175,10 +175,55 @@ jobs: echo "PR heads are now in current repo." # -------------------------------------------------------------------- - # STEP 3 — TRIGGER ANALYSIS WORKFLOWS + # STEP 3 — SKIP PRs WHOSE FILTERED RESULT ALREADY EXISTS + # -------------------------------------------------------------------- + - name: Filter PRs whose filtered result already exists + id: filter-prs + if: steps.fetch-prs.outputs.pr_count != '0' + env: + GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} + run: | + set -euo pipefail + repo_name=$(echo "${GITHUB_REPOSITORY}" | cut -d'/' -f2) + + # Collect all artifact names (paginate) + all_artifacts="" + page=1 + per_page=100 + while true; do + names=$(curl -s -H "Authorization: token $GH_TOKEN" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=${per_page}&page=${page}" \ + | jq -r '.artifacts[].name') + all_artifacts="${all_artifacts}"$'\n'"${names}" + [[ -z "$names" ]] || [[ $(echo "$names" | wc -l) -lt $per_page ]] && break + page=$((page + 1)) + done + + # Head filter artifact pattern: continuous-analysis-prs-filtered-results-*-{repo_name}-{head_sha} + results="[]" + total=$(jq 'length' prs_to_analyze.json) + for idx in $(seq 0 $((total - 1))); do + pr_num=$(jq -r ".[$idx].pr_number" prs_to_analyze.json) + head_sha=$(jq -r ".[$idx].head_sha" prs_to_analyze.json) + suffix="-${repo_name}-${head_sha}" + if echo "$all_artifacts" | grep -q "continuous-analysis-prs-filtered-results-.*${suffix}"; then + echo "Skip PR #${pr_num}: head filter artifact already exists (head=${head_sha})" + continue + fi + entry=$(jq -c ".[$idx]" prs_to_analyze.json) + results=$(echo "$results" | jq --argjson e "$entry" '. + [$e]') + done + + echo "$results" > prs_to_process.json + count=$(echo "$results" | jq 'length') + echo "prs_to_process_count=$count" >> $GITHUB_OUTPUT + echo "PRs to process (after skipping already-done): $count" + + # -------------------------------------------------------------------- + # STEP 4 — TRIGGER ANALYSIS WORKFLOWS # -------------------------------------------------------------------- - name: Run analysis workflows for PRs - if: steps.fetch-prs.outputs.pr_count != '0' # Only run if there are PRs to analyze + if: steps.filter-prs.outputs.prs_to_process_count != '' && steps.filter-prs.outputs.prs_to_process_count != '0' env: GH_TOKEN: ${{ secrets.ORG_WIDE_TOKEN }} BRANCH: ${{ env.BRANCH }} @@ -190,8 +235,8 @@ jobs: RUNNER_DISPATCH_TIMEOUT=${{ env.RUNNER_DISPATCH_TIMEOUT }} MAX_CONCURRENT=${{ env.MAX_CONCURRENT }} - # Get the total number of PRs to analyze - total_prs=$(jq 'length' prs_to_analyze.json) + # Use PRs that still need processing (skip already have filter result) + total_prs=$(jq 'length' prs_to_process.json) echo "Processing ${total_prs} PR(s) in batches of ${MAX_CONCURRENT}..." # Iterate over the PRs in batches of max concurrent @@ -208,10 +253,10 @@ jobs: # Iterate over the PRs in the batch (reuse dispatch_id from Step 1) for ((idx=batch_start; idx