Commit ac08acaf authored by Anas Nashif's avatar Anas Nashif
Browse files

ci: twister: use workflow_call for prep job 

Use workflow_call to allow for different type of nodes depending on
event.

On push, we do not need to use zephyr runners, GH runners are enough and
are much faster to deploy and start.

This resolves an issue where push jobs will have to be queued for a
longer time waiting for the prep step, once the prep step is done, we
will have to wait one more time in the queue for requested nodes.

This should speed up execution of push events in CI.

Signed-off-by: default avatarAnas Nashif <anas.nashif@intel.com>
parent 0a8b1669

.github/workflows/twister-prep.yaml

0 → 100644
+150 −0
Changes for .github/workflows/twister-prep.yaml: 150 added lines, 0 removed lines.
Original line number Diff line number Diff line 
name: Prep



on:

  workflow_call:

    outputs:

      subset:

        description: subset

        value: ${{ jobs.prep_push.outputs.subset }} || ${{ jobs.prep_pr.outputs.subset }}

      size:

        description: size

        value: ${{ jobs.prep_push.outputs.size }} || ${{ jobs.prep_pr.outputs.size }}

      fullrun:

        description: fullrun

        value: ${{ jobs.prep_push.outputs.fullrun }} || ${{ jobs.prep_pr.outputs.size }}



jobs:

  prep_pr:

    if: github.repository_owner == 'zephyrproject-rtos' && github.event_name == 'pull_request_target'

    runs-on:

      group: zephyr-runner-v2-linux-x64-4xlarge

    container:

      image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026

      options: '--entrypoint /bin/bash'

    outputs:

      subset: ${{ steps.output-services.outputs.subset }}

      size: ${{ steps.output-services.outputs.size }}

      fullrun: ${{ steps.output-services.outputs.fullrun }}

    env:

      MATRIX_SIZE: 10

      PUSH_MATRIX_SIZE: 20

      DAILY_MATRIX_SIZE: 80

      BSIM_OUT_PATH: /opt/bsim/

      BSIM_COMPONENTS_PATH: /opt/bsim/components

      TESTS_PER_BUILDER: 700

      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

      BASE_REF: ${{ github.base_ref }}

    steps:

      - name: Apply container owner mismatch workaround

        run: |

          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not

          #        match the container user UID because of the way GitHub

          #        Actions runner is implemented. Remove this workaround when

          #        GitHub comes up with a fundamental fix for this problem.

          git config --global --add safe.directory ${GITHUB_WORKSPACE}



      - name: Print cloud service information

        run: |

          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"

          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"

          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"



      - name: Clone cached Zephyr repository

        if: github.event_name == 'pull_request_target'

        continue-on-error: true

        run: |

          git clone --shared /repo-cache/zephyrproject/zephyr .

          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}



      - name: Checkout

        if: github.event_name == 'pull_request_target'

        uses: actions/checkout@v4

        with:

          ref: ${{ github.event.pull_request.head.sha }}

          fetch-depth: 0

          persist-credentials: false



      - name: Environment Setup

        if: github.event_name == 'pull_request_target'

        run: |

          git config --global user.email "bot@zephyrproject.org"

          git config --global user.name "Zephyr Bot"

          rm -fr ".git/rebase-apply"

          git rebase origin/${BASE_REF}

          git clean -f -d

          git log  --pretty=oneline | head -n 10

          west init -l . || true

          west config manifest.group-filter -- +ci,+optional

          west config --global update.narrow true

          west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)

          west forall -c 'git reset --hard HEAD'



          echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV



      - name: Generate Test Plan with Twister

        if: github.event_name == 'pull_request_target'

        id: test-plan

        run: |

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER

          if [ -s .testplan ]; then

            cat .testplan >> $GITHUB_ENV

          else

            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV

          fi

          rm -f testplan.json .testplan



      - name: Determine matrix size

        id: output-services

        run: |

          if [ -n "${TWISTER_NODES}" ]; then

            subset="[$(seq -s',' 1 ${TWISTER_NODES})]"

          else

            subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"

          fi

          size=${TWISTER_NODES}



          echo "subset=${subset}" >> $GITHUB_OUTPUT

          echo "size=${size}" >> $GITHUB_OUTPUT

          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT



  prep_push:

    if: github.repository_owner == 'zephyrproject-rtos' && (github.event_name == 'push' || github.event_name == 'schedule')

    runs-on: ubuntu-22.04

    outputs:

      subset: ${{ steps.output-services.outputs.subset }}

      size: ${{ steps.output-services.outputs.size }}

      fullrun: ${{ steps.output-services.outputs.fullrun }}

    env:

      MATRIX_SIZE: 10

      PUSH_MATRIX_SIZE: 20

      DAILY_MATRIX_SIZE: 80

      BSIM_OUT_PATH: /opt/bsim/

      BSIM_COMPONENTS_PATH: /opt/bsim/components

      TESTS_PER_BUILDER: 700

      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

      BASE_REF: ${{ github.base_ref }}

    steps:

      - name: Print cloud service information

        run: |

          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"

          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"

          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"



      - name: Determine matrix size

        id: output-services

        run: |

          if [ "${{github.event_name}}" = "push" ]; then

            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"

            size=${MATRIX_SIZE}

          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then

            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"

            size=${DAILY_MATRIX_SIZE}

          else

            size=0

          fi



          echo "subset=${subset}" >> $GITHUB_OUTPUT

          echo "size=${size}" >> $GITHUB_OUTPUT

          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

.github/workflows/twister.yaml

+1 −102
Changes for .github/workflows/twister.yaml: 1 added line, 102 removed lines.
Original line number Diff line number Diff line
@@ -21,108 +21,7 @@ concurrency: 


jobs:

  twister-build-prep:

    if: github.repository_owner == 'zephyrproject-rtos'

    runs-on:

      group: zephyr-runner-v2-linux-x64-4xlarge

    container:

      image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026

      options: '--entrypoint /bin/bash'

    outputs:

      subset: ${{ steps.output-services.outputs.subset }}

      size: ${{ steps.output-services.outputs.size }}

      fullrun: ${{ steps.output-services.outputs.fullrun }}

    env:

      MATRIX_SIZE: 10

      PUSH_MATRIX_SIZE: 20

      DAILY_MATRIX_SIZE: 80

      BSIM_OUT_PATH: /opt/bsim/

      BSIM_COMPONENTS_PATH: /opt/bsim/components

      TESTS_PER_BUILDER: 700

      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

      BASE_REF: ${{ github.base_ref }}

    steps:

      - name: Apply container owner mismatch workaround

        run: |

          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not

          #        match the container user UID because of the way GitHub

          #        Actions runner is implemented. Remove this workaround when

          #        GitHub comes up with a fundamental fix for this problem.

          git config --global --add safe.directory ${GITHUB_WORKSPACE}



      - name: Print cloud service information

        run: |

          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"

          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"

          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"



      - name: Clone cached Zephyr repository

        if: github.event_name == 'pull_request_target'

        continue-on-error: true

        run: |

          git clone --shared /repo-cache/zephyrproject/zephyr .

          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}



      - name: Checkout

        if: github.event_name == 'pull_request_target'

        uses: actions/checkout@v4

        with:

          ref: ${{ github.event.pull_request.head.sha }}

          fetch-depth: 0

          persist-credentials: false



      - name: Environment Setup

        if: github.event_name == 'pull_request_target'

        run: |

          git config --global user.email "bot@zephyrproject.org"

          git config --global user.name "Zephyr Bot"

          rm -fr ".git/rebase-apply"

          git rebase origin/${BASE_REF}

          git clean -f -d

          git log  --pretty=oneline | head -n 10

          west init -l . || true

          west config manifest.group-filter -- +ci,+optional

          west config --global update.narrow true

          west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)

          west forall -c 'git reset --hard HEAD'



          echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV



      - name: Generate Test Plan with Twister

        if: github.event_name == 'pull_request_target'

        id: test-plan

        run: |

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER

          if [ -s .testplan ]; then

            cat .testplan >> $GITHUB_ENV

          else

            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV

          fi

          rm -f testplan.json .testplan



      - name: Determine matrix size

        id: output-services

        run: |

          if [ "${{github.event_name}}" = "pull_request_target" ]; then

            if [ -n "${TWISTER_NODES}" ]; then

              subset="[$(seq -s',' 1 ${TWISTER_NODES})]"

            else

              subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"

            fi

            size=${TWISTER_NODES}

          elif [ "${{github.event_name}}" = "push" ]; then

            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"

            size=${MATRIX_SIZE}

          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then

            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"

            size=${DAILY_MATRIX_SIZE}

          else

            size=0

          fi

          echo "subset=${subset}" >> $GITHUB_OUTPUT

          echo "size=${size}" >> $GITHUB_OUTPUT

          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

    uses: ./.github/workflows/twister-prep.yaml



  twister-build:

    runs-on:

CRA Git | Maintained and supported by SUSTech CRA and CCSE