I'm encountering an issue with the "Leave a comment on the PR if workflow failed" job in our CI/CD pipeline. This job is part of the check_workflow_statuses workflow, which ensures all necessary tests pass before marking the PR as successful.
When the workflow fails, the job should leave a comment on the associated PR, notifying the contributor about the failure.
The job consistently fails, even though I've explicitly set pull-requests: write permissions in the workflow.
Here's the relevant portion of the workflow configuration:
name: Full-stack tests
permissions: read-all
on:
merge_group:
types: [checks_requested]
push:
branches:
- develop
- release-*
pull_request:
branches:
- develop
- release-*
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
e2e_and_acceptance_coverage:
name: Verify all e2e/acceptance tests are included
runs-on: ubuntu-22.04
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Check that all e2e and acceptance test files are captured in wdio.conf.js and core/tests/ci-test-suite-configs
run: python -m scripts.check_tests_are_captured_in_ci
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "The e2e/acceptance coverage test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
check_test_suites_to_run:
name: Compute which tests to run
runs-on: ubuntu-22.04
outputs:
TEST_SUITES_TO_RUN: ${{ stepspute_test_suites.outputs.TEST_SUITES_TO_RUN }}
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
with:
# This fetches all git refs, which is needed in the
# check_ci_test_suites_to_run script below.
fetch-depth: 0
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- id: compute_test_suites
name: Check test suites to run
env:
SHOULD_OUTPUT_ALL_TESTS: ${{ github.event_name != 'pull_request' || vars.RUN_SUITES_ON_CHANGED_FILES == 'false' }}
# Note that the script also writes the output to $GITHUB_OUTPUT.
run: |
TEST_SUITES_TO_RUN=$(python -m scripts.check_ci_test_suites_to_run --github_head_ref="HEAD" --github_base_ref="origin/${{ github.event.pull_request.base.ref }}" ${{ env.SHOULD_OUTPUT_ALL_TESTS == 'true' && '--output_all_test_suites' || '' }})
- name: Upload root files mapping as a GitHub artifact
uses: actions/upload-artifact@v4
with:
name: root-files-mapping
path: root-files-mapping.json
build:
name: Build the app, and store build files as an artifact
needs: [check_test_suites_to_run]
runs-on: ubuntu-22.04
if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.count > 0 ||
fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.count > 0 ||
fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.python.count > 0 }}
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Generate build files
uses: ./.github/actions/generate-build-files
e2e_test:
needs: [check_test_suites_to_run, build]
runs-on: ubuntu-22.04
if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.count > 0 }}
strategy:
matrix:
suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.suites }}
name: E2E (${{ matrix.suite.name }})
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Generate build files
uses: ./.github/actions/generate-build-files
- name: Install Chrome
uses: ./.github/actions/install-chrome
- name: Install ffmpeg for wdio videos
run: |
sudo apt-get update
sudo apt install ffmpeg
- name: Generate modified suite name for artifacts
id: generate_suite_name_for_artifacts
# Replace slashes in the filename with hyphens.
run: |
SUITE_NAME=${{ matrix.suite.name }}
echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT
- name: Run E2E Test ${{ matrix.suite.name }}
uses: oppia/retry@a9fb265410e634c84ee90f3e87f323fde0541037
with:
max_attempts: 2
substrings_indicating_flaky_execution: ${{ matrix.suite.flaky_indicators || '' }}
command: >
VIDEO_RECORDING_IS_ENABLED=0
xvfb-run -a --server-args="-screen 0, 1285x1000x24"
python -m scripts.run_e2e_tests --skip-install
--skip-build --suite=${{ matrix.suite.name }} --prod_env
- name: Uploading webdriverio-video as Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: webdriverio-video-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/webdriverio-video
- name: Uploading webdriverio screenshots as Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: webdriverio-screenshots-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/webdriverio-screenshots
- name: Uploading webpack bundles as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: webpack-bundles-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/oppia/build
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "An E2E test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
acceptance_test_with_python_installation:
needs: [check_test_suites_to_run, build]
runs-on: ubuntu-22.04
if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.python.count > 0 }}
strategy:
matrix:
suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.python.suites }}
name: Acceptance (${{ matrix.suite.name }})
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Generate build files
uses: ./.github/actions/generate-build-files
- name: Generate modified suite name for artifacts
id: generate_suite_name_for_artifacts
# Replace slashes in the filename with underscores.
run: |
SUITE_NAME=${{ matrix.suite.name }}
echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT
- name: Run Desktop Acceptance Test ${{ matrix.suite.name }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite.name }} --prod_env
- name: Uploading generated test to angular module mapping as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: generated-test-to-angular-module-mapping-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/oppia/core/tests/test-modules-mappings/acceptance/${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.txt
- name: Run Mobile Acceptance Test ${{ matrix.suite.name }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite.name }} --prod_env --mobile
- name: Uploading webpack bundles as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: webpack-bundles-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/oppia/build
- name: Uploading diff screenshots as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: diff-snapshots-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
# Note: The path must be kept in sync with the one specified in
# core/tests/puppeteer-acceptance-tests/utilities/common/puppeteer-utils.ts
path: /home/runner/work/oppia/oppia/core/tests/puppeteer-acceptance-tests/diff-snapshots
- name: Report failure if failed on oppia/oppia develop branch
if: ${{
failure() &&
github.event_name == 'push' &&
github.repository == 'oppia/oppia' &&
github.ref == 'refs/heads/develop'
}}
uses: ./.github/actions/send-webhook-notification
with:
message: "An acceptance test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
lighthouse_accessibility_test:
needs: [check_test_suites_to_run]
if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_accessibility.count > 0 }}
runs-on: ubuntu-22.04
strategy:
matrix:
suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_accessibility.suites }}
name: Lighthouse a11y (shard ${{ matrix.suite.name }})
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Install Chrome
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
uses: ./.github/actions/install-chrome
- name: Run Lighthouse accessibility checks shard ${{ matrix.suite.name }}
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
run: |
python -m scripts.run_lighthouse_tests --mode accessibility --pages ${{ join(matrix.suite.pages_to_run, ',') }}
shell: bash
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "A Lighthouse test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
lighthouse_performance_test:
needs: [check_test_suites_to_run, build]
runs-on: ubuntu-22.04
if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.count > 0 }}
strategy:
matrix:
suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.suites }}
name: Lighthouse perf (shard ${{ matrix.suite.name }})
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Generate build files
uses: ./.github/actions/generate-build-files
- name: Install Chrome
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
uses: ./.github/actions/install-chrome
- name: Generate modified suite name for artifacts
id: generate_suite_name_for_artifacts
# Replace slashes in the filename with underscores.
run: |
SUITE_NAME=${{ matrix.suite.name }}
echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT
- name: Run Lighthouse performance checks (shard ${{ matrix.suite.name }})
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
run: python -m scripts.run_lighthouse_tests --mode performance --skip_build --record_screen --pages ${{ join(matrix.suite.pages_to_run, ',') }}
- name: Uploading puppeteer video as artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: lhci-puppeteer-video-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
path: /home/runner/work/oppia/lhci-puppeteer-video/video.mp4
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "A Lighthouse performance test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
check_workflow_statuses:
# This job is needed because we cannot make each e2e/acceptance/lighthouse
# task a "required" check via GitHub, since different tasks may execute
# depending on which files are modified in the PR.
name: Check that all necessary tests pass
needs:
- check_test_suites_to_run
- e2e_test
- acceptance_test_with_python_installation
- lighthouse_accessibility_test
- lighthouse_performance_test
if: always()
runs-on: ubuntu-22.04
steps:
- name: Checkout repository so that local actions can be used
uses: actions/checkout@v4
- name: Merge develop and set up dependencies
uses: ./.github/actions/merge-develop-and-set-up-dependencies
- name: Check workflow status
uses: ./.github/actions/check-workflow-status
id: check_workflow_status
with:
jobs: ${{ toJson(needs) }}
- name: Leave a comment on the PR if workflow failed
if: ${{ steps.check_workflow_status.outputs.WORKFLOW_STATUS == 'failure' }}
uses: ./.github/actions/post-comment
with:
owner: ${{ github.event.repository.owner.login }}
repo: ${{ github.event.repository.name }}
issue_number: ${{ github.event.number }}
message: >