Commit 4c265418 authored by Anas Nashif's avatar Anas Nashif
Browse files

actions: run twister using github action 



This action replaces current buildkite workflow and uses github actions
to build and run tests in the zephyr tree using twister. The main
differences to current builtkite workflow:

- the action handles all 3 events: pull requests, push and schedule

- the action determines size of matrix (number of build hosts) based on
  the change with a minimum of 1 builder. If more tests are built/run
  due to changes to boards or tests/samples, the matrix size is
  increased. This will avoid timeouts when running over capacity due to
  board/test changes.

- We use ccache and store cache files on amazon S3 for more flexibility

- Results are collected per build host and merged in the final step and
  failures are posted into github action check runs.

- It runs on more powerful instances that can handle more load.
  Currently we have 10 build hosts per run (that can increase depending
  on number of tests run) and can deliver results within 1 hour.

- the action can deal with non code changes and will not allocate more
  than required to deal with changes to documentation and other files
  that do not require running twister

The goal long-term is better integrate this workflow with other actions
and not run unncessarily if other workflows have failed, for example, if
commit message is bogus, we should stop at that check, to avoid wasting
resources given that the commit message will have to be fixed anyways
which would later trigger another run on the same code.

Currently there is 1 open issue with this action related to a github
workflow bug where the final results are not posted to the same workflow
and might appear under other workflows. Github is working on this bug.

Signed-off-by: default avatarAnas Nashif <anas.nashif@intel.com>
parent 0a15b510

.github/workflows/twister.yaml

0 → 100644
+225 −0
Original line number Diff line number Diff line 
name: Run tests with twister



on:

  push:

    branches:

      - main

  pull_request_target:

    branches:

      - main

  schedule:

    # Run at 00:00 on Saturday

    - cron: '0 88888888 * * 6'



jobs:

  twister-build-prep:

    runs-on: zephyr_runner

    container:

      image: zephyrprojectrtos/ci:v0.18.4

      options: '--entrypoint /bin/bash'

    outputs:

      subset: ${{ steps.output-services.outputs.subset }}

      size: ${{ steps.output-services.outputs.size }}

    env:

      MATRIX_SIZE: 10

      DAILY_MATRIX_SIZE: 120

      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.13.1

      CLANG_ROOT_DIR: /usr/lib/llvm-12

      TESTS_PER_BUILDER: 700

      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

    steps:

      - name: Cancel Previous Runs

        uses: styfle/cancel-workflow-action@0.6.0

        with:

          access_token: ${{ github.token }}



      - name: checkout

        if: github.event_name == 'pull_request_target'

        uses: actions/checkout@v2

        with:

          ref: ${{ github.event.pull_request.head.sha }}

          fetch-depth: 0



      - name: west setup

        if: github.event_name == 'pull_request_target'

        run: |

          west init -l . || true

          west config --global update.narrow true

          west update 2>&1 1> west.update.log || west update 2>&1 1> west.update.log

          west forall -c 'git reset --hard HEAD'



      - name: Generate Test Plan with Twister

        if: github.event_name == 'pull_request_target'

        id: test-plan

        run: |

          sudo apt-get install -y bc

          git config --global user.email "bot@zephyrproject.org"

          git config --global user.name "Zephyr Bot"

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          ./scripts/ci/run_ci.sh -S -c -b ${{github.base_ref}} -r origin \

                  -p ${{github.event.pull_request.number}} -R ${COMMIT_RANGE}

          # remove all tests to be skipped

          grep -v skipped test_file.txt > no_skipped.txt

          # get number of tests

          lines=$(wc -l < no_skipped.txt)

          if [ "$lines" = 1 ]; then

            # no tests, so we need 0 nodes

            nodes=0

          else

            nodes=$(echo "${lines} / ${TESTS_PER_BUILDER}" | bc)

            if [ "${nodes}" = 0 ]; then

              # for less than TESTS_PER_BUILDER, we take at least 1 node

              nodes=1

            fi

          fi

          echo "::set-output name=calculated_matrix_size::${nodes}";

          rm test_file.txt no_skipped.txt



      - name: Determine matrix size

        id: output-services

        run: |

          if [ "${{github.event_name}}" = "pull_request_target" ]; then

            if [ -n "${{steps.test-plan.outputs.calculated_matrix_size}}" ]; then

              subset="[$(seq -s',' 1 ${{steps.test-plan.outputs.calculated_matrix_size}})]"

            else

              subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"

            fi

            size=${{ steps.test-plan.outputs.calculated_matrix_size }}

          elif [ "${{github.event_name}}" = "push" ]; then

            subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"

            size=${MATRIX_SIZE}

          else

            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"

            size=${DAILY_MATRIX_SIZE}

          fi

          echo "::set-output name=subset::${subset}";

          echo "::set-output name=size::${size}";





  twister-build:

    runs-on: zephyr_runner

    needs: twister-build-prep

    if: needs.twister-build-prep.outputs.size != 0

    container:

      image: zephyrprojectrtos/ci:v0.18.4

      options: '--entrypoint /bin/bash'

    strategy:

      fail-fast: false

      matrix:

        subset: ${{fromJSON(needs.twister-build-prep.outputs.subset)}}

    env:

      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.13.1

      CLANG_ROOT_DIR: /usr/lib/llvm-12

      DAILY_OPTIONS: ' --inline-logs -M -N --build-only --all --retry-failed 3 -v '

      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

    steps:

      - name: Update PATH for west

        run: |

          echo "$HOME/.local/bin" >> $GITHUB_PATH



      - name: checkout

        uses: actions/checkout@v2

        with:

          ref: ${{ github.event.pull_request.head.sha }}

          fetch-depth: 0



      - name: west setup

        run: |

          west init -l . || true

          west config --global update.narrow true

          west update 2>&1 1> west.update.log || west update 2>&1 1> west.update.log

          west forall -c 'git reset --hard HEAD'



      - name: Check Environment

        run: |

          cmake --version

          ${CLANG_ROOT_DIR}/bin/clang --version

          gcc --version

          ls -la

          echo "github.ref: ${{ github.ref }}"

          echo "github.base_ref: ${{ github.base_ref }}"

          echo "github.ref_name: ${{ github.ref_name }}"



      - name: Prepare ccache timestamp/data

        id: ccache_cache_timestamp

        shell: cmake -P {0}

        run: |

          string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)

          string(REPLACE "/" "_" repo ${{github.repository}})

          string(REPLACE "-" "_" repo2 ${repo})

          message("::set-output name=repo::${repo2}")



      - name: use cache

        id: cache-ccache

        uses: nashif/action-s3-cache@master

        with:

          key: ${{ steps.ccache_cache_timestamp.outputs.repo }}-${{ github.ref_name }}-${{github.event_name}}-${{ matrix.subset }}-ccache

          path: /github/home/.ccache

          aws-s3-bucket: ccache.zephyrproject.org

          aws-access-key-id: ${{ secrets.CCACHE_S3_ACCESS_KEY_ID }}

          aws-secret-access-key: ${{ secrets.CCACHE_S3_SECRET_ACCESS_KEY }}

          aws-region: us-east-2



      - name: ccache stats initial

        run: |

          test -d github/home/.ccache && mv github/home/.ccache /github/home/.ccache

          ccache -M 10G -s



      - if: github.event_name == 'push'

        name: Run Tests with Twister (Push)

        run: |

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          ./scripts/ci/run_ci.sh -c -b main -r origin -m ${{matrix.subset}} \

                    -M ${{ strategy.job-total }}



      - if: github.event_name == 'pull_request_target'

        name: Run Tests with Twister (Pull Request)

        run: |

          git config --global user.email "bot@zephyrproject.org"

          git config --global user.name "Zephyr Builder"

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          ./scripts/ci/run_ci.sh -c -b ${{github.base_ref}} -r origin \

                     -m ${{matrix.subset}} -M  ${{ strategy.job-total }} \

                     -p ${{github.event.pull_request.number}} -R ${COMMIT_RANGE}



      - if: github.event_name == 'schedule'

        name: Run Tests with Twister (Daily)

        run: |

          export ZEPHYR_BASE=${PWD}

          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr

          ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} ${DAILY_OPTIONS}



      - name: ccache stats post

        run: |

          ccache -s



      - name: Upload Unit Test Results

        if: always()

        uses: actions/upload-artifact@v2

        with:

          name: Unit Test Results (Subset ${{ matrix.subset }})

          path: twister-out/twister.xml



  twister-test-results:

    name: "Publish Unit Tests Results"

    needs: twister-build

    runs-on: ubuntu-latest

      # the build-and-test job might be skipped, we don't need to run this job then

    if: success() || failure()



    steps:

      - name: Download Artifacts

        uses: actions/download-artifact@v2

        with:

          path: artifacts



      - name: Publish Unit Test Results

        uses: EnricoMi/publish-unit-test-result-action@v1

        with:

          check_name: Unit Test Results

          github_token: ${{ secrets.GITHUB_TOKEN }}

          files: "**/twister.xml"

          comment_mode: off

scripts/ci/run_ci.sh

+29 −10
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 


set -xe



twister_options=" --inline-logs -M -N -v --integration"

twister_options=" --clobber-output --inline-logs -M -N -v --integration"



west_commands_results_file="./pytest_out/west_commands.xml"
@@ -99,9 +99,10 @@ function build_test_file() { 
		./scripts/zephyr_module.py --twister-out module_tests.args

		./scripts/ci/get_twister_opt.py --commits ${commit_range}



		if [ -s modified_tags.args ]; then

			twister_exclude_tag_opt="+modified_tags.args"

		fi

		# disabled for now due to a bug

		#if [ -s modified_tags.args ]; then

		#	twister_exclude_tag_opt="+modified_tags.args"

		#fi



		if [ -s modified_boards.args ]; then

			${twister} ${twister_options} ${twister_exclude_tag_opt} \
@@ -136,10 +137,23 @@ function build_test_file() { 
	tail -n +2 test_file_tests.txt > test_file_tests_in.txt

	tail -n +2 test_file_boards.txt > test_file_boards_in.txt



	echo -n "Full: "

	wc -l test_file_full.txt

	echo -n "Arch: "

	wc -l test_file_archs.txt

	echo -n "Tests: "

	wc -l test_file_tests.txt

	echo -n "Boards: "

	wc -l test_file_boards.txt





	echo "test,arch,platform,status,extra_args,handler,handler_time,ram_size,rom_size" \

		> test_file.txt

	cat test_file_full_in.txt test_file_archs_in.txt test_file_tests_in.txt \

		test_file_boards_in.txt >> test_file.txt



	echo -n "Total: "

	wc -l test_file.txt

}



function west_setup() {
@@ -155,7 +169,7 @@ function west_setup() { 
}





while getopts ":p:m:b:r:M:cfslR:" opt; do

while getopts ":p:m:b:r:M:cSfslR:" opt; do

	case $opt in

		c)

			echo "Execute CI" >&2
@@ -190,6 +204,9 @@ while getopts ":p:m:b:r:M:cfslR:" opt; do 
			echo "Base Branch: $OPTARG" >&2

			branch=$OPTARG

			;;

		S)

			output_plan=1

			;;

		r)

			echo "Remote: $OPTARG" >&2

			remote=$OPTARG
@@ -215,9 +232,11 @@ if [ -n "$main_ci" ]; then 
		commit_range=$remote/${branch}..HEAD

		echo "Commit range:" ${commit_range}

	fi



	if [ -n "$range" ]; then

		commit_range=$range

	fi



	source zephyr-env.sh

	twister="${ZEPHYR_BASE}/scripts/twister"
@@ -234,16 +253,16 @@ if [ -n "$main_ci" ]; then 
	fi



	if [ -n "$pull_request_nr" ]; then

		$short_git_log $remote/${branch}

		# Now let's pray this script is being run from a

		# different location

# https://stackoverflow.com/questions/3398258/edit-shell-script-while-its-running

		git rebase $remote/${branch}

		$short_git_log $commit_range

	fi

	$short_git_log



	build_test_file



	if [ -n "${output_plan}" ]; then

		exit 0

	fi



	echo "+++ run twister"



	# Run a subset of tests based on matrix size

CRA Git | Maintained and supported by SUSTech CRA and CCSE