name: Integration Tests CUDA

on:
  workflow_call:
    inputs:
      matrix:
        required: true
        type: string

jobs:
  integration-tests-nvidia:
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    strategy:
      matrix:
        runner: ${{ fromJson(inputs.matrix) }}
    env:
      RUNNER_TYPE: ${{ matrix.runner[0] }}
      TRITON_BUILD_WITH_CCACHE: "true"
      TRITON_BUILD_WITH_CLANG_LLD: "TRUE"
      TRITON_USE_ASSERT_ENABLED_LLVM: "TRUE"
      TRITON_DISABLE_LINE_INFO: 1
      PROTON_SKIP_PC_SAMPLING_TEST: 1
      PYTHON: "python3"
      CCACHE_COMPRESS: "true"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: "true"
      - name: Compute cache keys
        id: cache-key
        run: |
          llvm_file="cmake/llvm-hash.txt"
          nvidia_file="cmake/nvidia-toolchain-version.json"
          json_file="cmake/json-version.txt"

          # Check if files exist before proceeding
          if [[ ! -f "$llvm_file" || ! -f "$nvidia_file" || ! -f "$json_file" ]]; then
            echo "Error: Required dependency files are missing."
            exit 1
          fi

          # Process the files if they exist
          echo "llvm=$(cat $llvm_file | cut -c 1-8)" >> $GITHUB_OUTPUT
          echo "nvidia=$(sha256sum $nvidia_file | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
          echo "json=$(cat $json_file)" >> $GITHUB_OUTPUT
          echo "datetime=$(date -u -Iseconds)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Cache build dependencies
        uses: actions/cache@v4
        with:
          # Note that we cannot use environment variables here given there is
          # no shell to interpret them in the paths.
          path: |
            ~/.triton/llvm
            ~/.triton/nvidia
            ~/.triton/json
          key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.cache-key.outputs.llvm }}-nvidia-${{ steps.cache-key.outputs.nvidia }}-json-${{ steps.cache-key.outputs.json }}
      - # Cache ~/.cache/ccache to speed up compilation.
        #
        # On branch `main` we always start from an empty cache, i.e. we skip the
        # "restore" step.  This is to prevent the caches from accumulating stale
        # files over time.
        name: Restore cache of ccache and Triton compilation artifacts
        id: restore-build-cache
        if: github.ref != 'refs/heads/main'
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.ccache
          # Restore the most recent cache entry.
          restore-keys: |
            triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-
            triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-
          # We expect this cache key never to hit and for us to fall back
          # unconditionally to the restore-key, so it doesn't actually matter
          # what we put here (so long as it doesn't hit an existing key).
          key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
      - name: Inspect cache directories
        run: |
          mkdir -p ~/.triton
          du -h -d 1 ~/.triton

          mkdir -p ~/.ccache
          du -h -d 1 ~/.ccache
      - name: Update PATH
        run: |
          echo "$HOME/.local/bin" >> $GITHUB_PATH
      - name: Install Triton
        env:
          CUDA_HOME: "/usr/local/cuda"
        run: |
          if [ "${{ matrix.runner[0] }}" == "nvidia-gb200" ]; then
            source /venv/bin/activate
          fi
          echo "PATH is '$PATH'"
          ccache --zero-stats
          make dev-install
      - name: CCache Stats
        run: ccache --print-stats
      - name: Run lit tests
        run: make test-lit
      - name: Run python tests on CUDA
        run: |
          if [ "${{ matrix.runner[0] }}" == "nvidia-gb200" ]; then
            source /venv/bin/activate
          fi
          make test-unit
      - name: Run interpreter tests
        if: ${{ matrix.runner[0] == 'nvidia-h100' }}
        run: make test-interpret
      - name: Run regression tests
        run: |
          if [ "${{ matrix.runner[0] }}" == "nvidia-gb200" ]; then
            source /venv/bin/activate
          fi
          make test-regression
      - name: Run C++ unittests
        run: make test-cpp
      - name: Run Proton tests
        if: ${{ matrix.runner[0] != 'nvidia-gb200' }}
        run: make test-proton
      - name: Inspect cache directories
        run: |
          mkdir -p ~/.triton
          du -h -d 1 ~/.triton

          mkdir -p ~/.ccache
          du -h -d 1 ~/.ccache
      - # If we're on branch `main`, save the ccache Triton compilation artifacts
        # to the cache so they can be used by other (non-main) CI runs.
        #
        # (It wouldn't be a problem to save the cache on every run, because github
        # evicts cache entries LRU, but maybe this saves a bit of time in CI.)
        name: Save ccache and Triton compilation artifacts to cache
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@v4
        with:
          path: |
            ~/.ccache
          key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
