Enable more Launchable integration in bootstraptest and test-all (#12255)

6 files changed, 119 insertions, 41 deletions

diff --git a/.github/actions/launchable/setup/action.yml b/.github/actions/launchable/setup/action.yml
index 2d92cb7d99..6ab8763df9 100644
--- a/.github/actions/launchable/setup/action.yml
+++ b/.github/actions/launchable/setup/action.yml
@@ -3,11 +3,6 @@ description: >-
   Install the required dependencies and execute the necessary Launchable commands for test recording
 
 inputs:
-  report-path:
-    default: launchable_reports.json
-    required: true
-    description: The file path of the test report for uploading to Launchable
-
   os:
     required: true
     description: The operating system that CI runs on. This value is used in Launchable flavor.
@@ -38,18 +33,27 @@ inputs:
       Directory to (re-)checkout source codes. Launchable retrieves the commit information
       from the directory.
 
-  launchable-workspace:
-    required: true
-    default: ${{ github.event.repository.name }}
-    description: >-
-      A workspace name in Launchable
-
   test-task:
-    required: true
+    required: false
     default: ${{ matrix.test_task }}
     description: >-
-      A test task that determine which tests are executed.
+      Specifies a single test task to be executed.
       This value is used in the Launchable flavor.
+      Either 'test-task' or 'multi-test-tasks' must be configured.
+
+  test-tasks:
+    required: false
+    default: '[]'
+    description: >-
+      Specifies an array of multiple test tasks to be executed.
+      For example: '["test", "test-all"]'.
+      If you want to run a single test task, use the 'test-task' input instead.
+
+  is-yjit:
+    required: false
+    default: 'false'
+    description: >-
+      Whether this workflow is executed on YJIT.
 
 runs:
   using: composite
@@ -61,11 +65,14 @@ runs:
       shell: bash
       if: >-
         ${{
-        (github.repository == 'ruby/ruby' ||
-        (github.repository != 'ruby/ruby' && env.LAUNCHABLE_TOKEN)) &&
-        (inputs.test-task == 'check' ||
-        inputs.test-task == 'test-all' ||
-        inputs.test-task == 'test')
+        (github.repository == 'ruby/ruby'
+          || (github.repository != 'ruby/ruby'
+          && env.LAUNCHABLE_TOKEN))
+        && (inputs.test-task == 'check'
+          || inputs.test-task == 'test-all'
+          || inputs.test-task == 'test'
+          || contains(fromJSON(inputs.test-tasks), 'test-all')
+          || contains(fromJSON(inputs.test-tasks), 'test'))
         }}
 
     # Launchable CLI requires Python and Java.
@@ -83,6 +90,20 @@ runs:
         java-version: '17'
       if: steps.enable-launchable.outputs.enable-launchable
 
+    - name: Set global vars
+      id: global
+      shell: bash
+      run: |
+        test_all_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test-all' || contains(fromJSON(inputs.test-tasks), 'test-all') }}"
+        btest_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test' || contains(fromJSON(inputs.test-tasks), 'test') }}"
+        echo test_all_enabled="${test_all_enabled}" >> $GITHUB_OUTPUT
+        echo btest_enabled="${btest_enabled}" >> $GITHUB_OUTPUT
+        echo test_all_session_file='launchable_test_all_session.txt' >> $GITHUB_OUTPUT
+        echo btest_session_file='launchable_btest_session.txt' >> $GITHUB_OUTPUT
+        echo test_all_report_file='launchable_test_all_report.json' >> $GITHUB_OUTPUT
+        echo btest_report_file='launchable_btest_report.json' >> $GITHUB_OUTPUT
+      if: steps.enable-launchable.outputs.enable-launchable
+
     - name: Set environment variables for Launchable
       shell: bash
       run: |
@@ -92,7 +113,7 @@ runs:
         : # The following envs are necessary in Launchable tokenless authentication.
         : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L20
         echo "LAUNCHABLE_ORGANIZATION=${{ github.repository_owner }}" >> $GITHUB_ENV
-        echo "LAUNCHABLE_WORKSPACE=${{ inputs.launchable-workspace }}" >> $GITHUB_ENV
+        echo "LAUNCHABLE_WORKSPACE=${{ github.event.repository.name }}" >> $GITHUB_ENV
         : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L71
         echo "GITHUB_PR_HEAD_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_ENV
         echo "LAUNCHABLE_TOKEN=${{ inputs.launchable-token }}" >> $GITHUB_ENV
@@ -121,20 +142,51 @@ runs:
         : #
         : # FIXME: Need to fix `WARNING: Failed to process a change to a file`.
         : # https://github.com/launchableinc/cli/issues/786
-        launchable record build --name ${github_ref}_${GITHUB_PR_HEAD_SHA}
-        echo "TESTS=${TESTS} --launchable-test-reports=${{ inputs.report-path }}" >> $GITHUB_ENV
+        build_name="${github_ref}_${GITHUB_PR_HEAD_SHA}"
+        test_opts="${{ inputs.test-opts }}"
+        test_opts="${test_opts// /}"
+        test_opts="${test_opts//=/:}"
+        test_all_test_suite='test-all'
+        btest_test_suite='btest'
+        if [ "${{ inputs.is-yjit }}" = "true" ]; then
+          test_all_test_suite="yjit-${test_all_test_suite}"
+          btest_test_suite="yjit-${btest_test_suite}"
+        fi
+        launchable record build --name "${build_name}"
+        if [ "${test_all_enabled}" = "true" ]; then
+          launchable record session \
+            --build "${build_name}" \
+            --flavor os=${{ inputs.os }} \
+            --flavor test_task=${{ inputs.test-task }} \
+            --flavor test_opts=${test_opts} \
+            --test-suite ${test_all_test_suite} \
+            > "${test_all_session_file}"
+          echo "TESTS=${TESTS} --launchable-test-reports=${test_all_report_file}" >> $GITHUB_ENV
+        fi
+        if [ "${btest_enabled}" = "true" ]; then
+          launchable record session \
+            --build "${build_name}" \
+            --flavor os=${{ inputs.os }} \
+            --flavor test_task=${{ inputs.test-task }} \
+            --flavor test_opts=${test_opts} \
+            --test-suite ${btest_test_suite} \
+            > "${btest_session_file}"
+          echo "BTESTS=${BTESTS} --launchable-test-reports=${btest_report_file}" >> $GITHUB_ENV
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
+      env:
+        test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
+        btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
+        btest_session_file: ${{ steps.global.outputs.btest_session_file }}
+        test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
+        btest_report_file: ${{ steps.global.outputs.btest_report_file }}
 
     - name: Variables to report Launchable
       id: variables
       shell: bash
       run: |
         set -x
-        : # flavor
-        test_opts="${{ inputs.test-opts }}"
-        test_opts="${test_opts// /}"
-        test_opts="${test_opts//=/:}"
-        echo test-opts="$test_opts" >> $GITHUB_OUTPUT
         : # report-path from srcdir
         if [ "${srcdir}" = "${{ github.workspace }}" ]; then
           dir=
@@ -142,13 +194,22 @@ runs:
           # srcdir must be equal to or under workspace
           dir=$(echo ${srcdir:+${srcdir}/} | sed 's:[^/][^/]*/:../:g')
         fi
-        report_path="${dir}${builddir:+${builddir}/}${report_path}"
-        echo report-path="${report_path}" >> $GITHUB_OUTPUT
+        if [ "${test_all_enabled}" = "true" ]; then
+          test_report_path="${dir}${builddir:+${builddir}/}${test_all_report_file}"
+          echo test_report_path="${test_report_path}" >> $GITHUB_OUTPUT
+        fi
+        if [ "${btest_enabled}" = "true" ]; then
+          btest_report_path="${dir}${builddir:+${builddir}/}${btest_report_file}"
+          echo btest_report_path="${btest_report_path}" >> $GITHUB_OUTPUT
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
       env:
         srcdir: ${{ inputs.srcdir }}
         builddir: ${{ inputs.builddir }}
-        report_path: ${{ inputs.report-path }}
+        test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
+        btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
+        btest_report_file: ${{ steps.global.outputs.btest_report_file }}
 
     - name: Record test results in Launchable
       uses: gacts/run-and-post-run@674528335da98a7afc80915ff2b4b860a0b3553a # v1.4.0
@@ -156,10 +217,25 @@ runs:
         shell: bash
         working-directory: ${{ inputs.srcdir }}
         post: |
-          : # record
-          launchable record tests --flavor os=${{ inputs.os }} --flavor test_task=${{ inputs.test-task }} --flavor test_opts=${test_opts} raw ${report_path}
-          rm -f ${report_path}
+          [[ "${test_all_enabled}" = "true" ]] && \
+            launchable record tests \
+              --session "$(cat "${test_all_session_file}")" \
+              raw "${test_report_path}" || true
+
+          [[ "${btest_enabled}" = "true" ]] && \
+            launchable record tests \
+              --session "$(cat "${btest_session_file}")" \
+              raw "${btest_report_path}" || true
+
+          rm -f "${test_all_session_file}"
+          rm -f "${btest_session_file}"
+          rm -f "${test_report_path}"
+          rm -f "${btest_report_path}"
       if: ${{ always() && steps.enable-launchable.outputs.enable-launchable }}
       env:
-        test_opts: ${{ steps.variables.outputs.test-opts }}
-        report_path: ${{ steps.variables.outputs.report-path }}
+        test_report_path: ${{ steps.variables.outputs.test_report_path }}
+        btest_report_path: ${{ steps.variables.outputs.btest_report_path }}
+        test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
+        btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
+        btest_session_file: ${{ steps.global.outputs.btest_session_file }}
diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml
index 37cbfa5f38..e03a8d2838 100644
--- a/.github/workflows/mingw.yml
+++ b/.github/workflows/mingw.yml
@@ -123,12 +123,10 @@ jobs:
         uses: ./.github/actions/launchable/setup
         with:
           os: windows-2022
-          # If we support new test task, we need to change this test-opts.
-          test-opts: --retry --job-status=normal --show-skip --timeout-scale=1.5
-            ${{ matrix.test-all-opts }}
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
+          test-tasks: '["test", "test-all"]'
         continue-on-error: true
 
       - name: test
diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml
index 12e6b94b53..320534d99c 100644
--- a/.github/workflows/rjit.yml
+++ b/.github/workflows/rjit.yml
@@ -83,12 +83,11 @@ jobs:
         uses: ./.github/actions/launchable/setup
         with:
           os: ubuntu-22.04
-          test-task: test
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
-          launchable-workspace: ruby-make-btest
           test-opts: ${{ matrix.run_opts }}
+          test-tasks: '["test", "test-all"]'
         continue-on-error: true
 
       - name: make test
@@ -103,8 +102,10 @@ jobs:
           RUN_OPTS: ${{ matrix.run_opts }}
 
       - name: make test-all
-        run: |
-          $SETARCH make -s test-all RUN_OPTS="$RUN_OPTS"
+        run: >-
+          $SETARCH make -s test-all
+          RUN_OPTS="$RUN_OPTS"
+          ${TESTS:+TESTS="$TESTS"}
         timeout-minutes: 60
         env:
           GNUMAKEFLAGS: ''
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 619f9c00af..90c8af82d6 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -181,6 +181,7 @@ jobs:
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
+          test-task: ${{ matrix.test_task || 'check' }}
         continue-on-error: true
         if: ${{ matrix.test_task != 'test-bundled-gems' }}
 
diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml
index e93981f379..b322abbfa1 100644
--- a/.github/workflows/yjit-macos.yml
+++ b/.github/workflows/yjit-macos.yml
@@ -124,6 +124,7 @@ jobs:
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
+          is-yjit: true
         continue-on-error: true
 
       - name: make ${{ matrix.test_task }}
diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml
index 7b7ec644a2..f1c62c6c77 100644
--- a/.github/workflows/yjit-ubuntu.yml
+++ b/.github/workflows/yjit-ubuntu.yml
@@ -177,6 +177,7 @@ jobs:
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
+          is-yjit: true
         continue-on-error: true
 
       - name: make ${{ matrix.test_task }}