rsnext/.github/workflows/nextjs-integration-test.yml

# Reusable workflow to execute certain version of Next.js integration tests
# with turbopack.
#
# Refer test.yml for how this workflow is being initialized
# - Workflow can specify `inputs.version` to specify which version of next.js to use, otherwise will use latest release version.
name: Turbopack Next.js integration test

on:
  workflow_call:
    inputs:
      # Allow to specify Next.js version to run integration test against.
      # If not specified, will use latest release version including canary.
      version:
        type: string
      # The base of the test results to compare against. If not specified, will try to compare with latest main branch's test results.
      diff_base:
        type: string
        default: 'main'
      force_post_to_slack:
        type: boolean
      # Skip posting to slack regardless of the conditions.
      skip_post_to_slack:
        type: boolean
        default: false

# Workflow-common env variables
env:
  # Enabling backtrace will makes snapshot tests fail
  RUST_BACKTRACE: 0
  NEXT_TELEMETRY_DISABLED: 1
  TEST_CONCURRENCY: 6
  DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
  DATADOG_TRACE_NEXTJS_TEST: 'true'
  DD_ENV: 'ci'
  # Turbopack specific customization for the test runner
  TURBOPACK: 1
  __INTERNAL_CUSTOM_TURBOPACK_BINDINGS: ${{ github.workspace }}/packages/next-swc/native/next-swc.linux-x64-gnu.node
  NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ github.workspace }}/integration-test-data/test-results/main/failed-test-path-list.json
  NEXT_TEST_CONTINUE_ON_ERROR: TRUE
  NEXT_E2E_TEST_TIMEOUT: 240000
  NEXT_TEST_JOB: 1

jobs:
  # First, build next-dev and Next.js both to execute across tests.
  setup_nextjs:
    name: Setup Next.js build
    uses: ./.github/workflows/setup-nextjs-build.yml
    with:
      version: ${{ inputs.version }}

  # Actual test scheduling. These jobs mimic the same jobs in Next.js repo,
  # which we do allow some of duplications to make it easier to update if upstream changes.
  # Refer build_and_test.yml in the Next.js repo for more details.
  test-dev:
    # This job name is being used in github action to collect test results. Do not change it, or should update
    # ./.github/actions/next-integration-stat to match the new name.
    name: Next.js integration test (Development)
    # Currently it is possible test grouping puts large number of failing tests suites in a single group,
    # which ends up job timeouts. Temporarily relieve the timeout until we make progresses on the failing suites.
    # ref: https://github.com/vercel/turbo/pull/5668
    # timeout-minutes: 180
    runs-on:
      - 'self-hosted'
      - 'linux'
      - 'x64'
      - 'metal'

    needs: [setup_nextjs]
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3, 4, 5, 6]

    steps:
      - uses: actions/cache/restore@v3
        id: restore-build
        with:
          path: ./*
          key: ${{ inputs.version }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt}}-${{ github.run_number }}
          fail-on-cache-miss: true

      - name: Enable corepack and install yarn
        run: |
          corepack enable
          corepack prepare --activate yarn@1.22.19          

      - name: Run test/development
        run: |
          ls $NEXT_TEST_SKIP_RETRY_MANIFEST
          sudo npx playwright install-deps && pnpm playwright install
          NEXT_TEST_MODE=dev TURBOPACK=1 node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --type development
          ls test/turbopack-test-junit-report          
        # It is currently expected to fail some of next.js integration test, do not fail CI check.
        continue-on-error: true
        env:
          # marker to parse log output, do not delete / change.
          NEXT_INTEGRATION_TEST: true

      - name: Upload test reports artifact
        uses: actions/upload-artifact@v3
        with:
          name: Test trace reports
          path: |
            test/turbopack-test-junit-report            

  test-integration:
    name: Next.js integration test (Integration)
    needs: [setup_nextjs]
    runs-on:
      - 'self-hosted'
      - 'linux'
      - 'x64'
      - 'metal'

    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]

    steps:
      - uses: actions/cache/restore@v3
        id: restore-build
        with:
          path: ./*
          key: ${{ inputs.version }}-${{ github.sha }}
          fail-on-cache-miss: true

      - name: Enable corepack and install yarn
        run: |
          corepack enable
          corepack prepare --activate yarn@1.22.19          

      - name: Run test/integration
        run: |
          sudo npx playwright install-deps && pnpm playwright install
          TURBOPACK=1 node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --type integration
          ls test/turbopack-test-junit-report          
        continue-on-error: true
        env:
          NEXT_INTEGRATION_TEST: true

      - name: Upload test reports artifact
        uses: actions/upload-artifact@v3
        with:
          name: Test trace reports
          path: |
            test/turbopack-test-junit-report            

  # Collect integration test results from execute_tests,
  # Store it as github artifact for next step to consume.
  collect_nextjs_integration_stat:
    needs: [test-dev, test-integration]
    name: Next.js integration test status report
    runs-on:
      - 'self-hosted'
      - 'linux'
      - 'x64'
      - 'metal'

    if: always()
    permissions:
      pull-requests: write
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Collect integration test stat
        uses: ./.github/actions/next-integration-stat
        with:
          diff_base: ${{ inputs.diff_base }}
        env:
          NEXT_TURBO_FORCE_SLACK_UPDATE: '${{ inputs.force_post_to_slack }}'
          NEXT_TURBO_FORCE_SKIP_SLACK_UPDATE: '${{ inputs.skip_post_to_slack }}'

      - name: Store artifacts
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: |
            nextjs-test-results.json
            failed-test-path-list.json
            passed-test-path-list.json
            slack-payload.json            

  upload_test_trace:
    needs: [test-dev, test-integration]
    name: Upload test trace to datadog
    runs-on:
      - 'self-hosted'
      - 'linux'
      - 'x64'
      - 'metal'

    if: always()
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Download test reports artifact
        id: download-test-reports
        uses: actions/download-artifact@v3
        with:
          name: Test trace reports
          path: test
      - name: Upload test trace to datadog
        run: |
          npm install -g @datadog/datadog-ci@2.18.1
          ls -al ./test/*.xml
          # We'll tag this to the Turbopack service, not the next.js
          DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack.daily --service Turbopack test          

      # For the debugging purpose, upload the test trace to github artifact.
      # So we can manually analyze test reports.
      - name: Upload test reports artifact
        uses: actions/upload-artifact@v3
        with:
          name: Merged test trace reports
          path: |
            nextjs-test-result-junit.xml
            ./**/*.xml