Commit 4f637258 authored by Lukasz Fundakowski's avatar Lukasz Fundakowski Committed by Benjamin Cabé
Browse files

twister: refactor twister_main to simplify the system tests 



Refactored twister_main.py module to simplify the code and system tests.
Removed the need to patch `sys.argv` in blackbox tests.

Signed-off-by: default avatarLukasz Fundakowski <lukasz.fundakowski@nordicsemi.no>
parent 654d794d

scripts/pylib/twister/twisterlib/twister_main.py

+19 −5
Original line number Diff line number Diff line
@@ -9,11 +9,17 @@ import os 
import shutil

import sys

import time

from collections.abc import Sequence



import colorama

from colorama import Fore

from twisterlib.coverage import run_coverage

from twisterlib.environment import TwisterEnv

from twisterlib.environment import (

    TwisterEnv,

    add_parse_arguments,

    parse_arguments,

    python_version_guard,

)

from twisterlib.hardwaremap import HardwareMap

from twisterlib.log_helper import close_logging, setup_logging

from twisterlib.package import Artifacts
@@ -27,7 +33,7 @@ def init_color(colorama_strip): 
    colorama.init(strip=colorama_strip)





def twister(options: argparse.Namespace, default_options: argparse.Namespace):

def twister(options: argparse.Namespace, default_options: argparse.Namespace) -> int:

    start_time = time.time()



    # Configure color output
@@ -230,9 +236,17 @@ def twister(options: argparse.Namespace, default_options: argparse.Namespace): 
    return 0





def main(options: argparse.Namespace, default_options: argparse.Namespace):

def main(argv: Sequence[str] | None = None) -> int:

    """Main function to run twister."""

    try:

        return_code = twister(options, default_options)

        python_version_guard()



        parser = add_parse_arguments()

        options = parse_arguments(parser, argv)

        default_options = parse_arguments(parser, [], on_init=False)

        return twister(options, default_options)

    finally:

        close_logging()

    return return_code
 
        if (os.name != "nt") and os.isatty(1):

            # (OS is not Windows) and (stdout is interactive)

            os.system("stty sane <&1")

scripts/tests/twister_blackbox/test_device.py

+13 −42
Original line number Diff line number Diff line
@@ -6,7 +6,6 @@ 
Blackbox tests for twister's command line functions related to test filtering.

"""



import importlib

from unittest import mock

import os

import pytest
@@ -14,63 +13,35 @@ import sys 
import re



# pylint: disable=no-name-in-module

from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock

from conftest import TEST_DATA, suite_filename_mock

from twisterlib.testplan import TestPlan

from twisterlib.twister_main import main as twister_main





class TestDevice:

    TESTDATA_1 = [

        (

            1234,

        ),

        (

            4321,

        ),

        (

            1324,

        )

    ]



    @classmethod

    def setup_class(cls):

        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')

        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)

        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)

        cls.twister_module = importlib.util.module_from_spec(cls.spec)



    @classmethod

    def teardown_class(cls):

        pass



    @pytest.mark.parametrize(

        'seed',

        TESTDATA_1,

        ids=[

            'seed 1234',

            'seed 4321',

            'seed 1324'

        ],

        [1234, 4321, 1324],

    )



    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)

    def test_seed(self, capfd, out_path, seed):

        test_platforms = ['native_sim']

        path = os.path.join(TEST_DATA, 'tests', 'seed_native_sim')

        args = ['--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',] + \

               ['--seed', f'{seed[0]}'] + \

               [val for pair in zip(

                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(SystemExit) as sys_exit:

            self.loader.exec_module(self.twister_module)

        args = [

            '--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',

            '--seed', f'{seed}',

            *[val for pair in zip(['-p'] * len(test_platforms), test_platforms) for val in pair]

        ]



        return_value = twister_main(args)



        out, err = capfd.readouterr()

        sys.stdout.write(out)

        sys.stderr.write(err)



        assert str(sys_exit.value) == '1'

        assert return_value == 1



        expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed[0])

        assert re.search(expected_line, err)
 
        expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed)

        assert re.search(expected_line, err), f'Regex not found: r"{expected_line}"'

scripts/tests/twister_blackbox/test_disable.py

+11 −26
Original line number Diff line number Diff line
@@ -6,7 +6,6 @@ 
Blackbox tests for twister's command line functions related to disable features.

"""



import importlib

import pytest

from unittest import mock

import os
@@ -14,8 +13,9 @@ import sys 
import re



# pylint: disable=no-name-in-module

from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock

from conftest import TEST_DATA, suite_filename_mock

from twisterlib.testplan import TestPlan

from twisterlib.twister_main import main as twister_main





@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
@@ -41,30 +41,17 @@ class TestDisable: 
            os.path.join(TEST_DATA, 'tests', 'always_warning'),

            ['qemu_x86'],

            '--disable-warnings-as-errors',

            '0'

            0

        ),

        (

            os.path.join(TEST_DATA, 'tests', 'always_warning'),

            ['qemu_x86'],

            '-v',

            '1'

            1

        ),

    ]





    @classmethod

    def setup_class(cls):

        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')

        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)

        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)

        cls.twister_module = importlib.util.module_from_spec(cls.spec)





    @classmethod

    def teardown_class(cls):

        pass





    @pytest.mark.parametrize(

        'test_path, test_platforms, flag, expected, expected_none',

        TESTDATA_1,
@@ -82,15 +69,14 @@ class TestDisable: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

            pytest.raises(SystemExit) as sys_exit:

            self.loader.exec_module(self.twister_module)



        return_value = twister_main(args)



        out, err = capfd.readouterr()

        sys.stdout.write(out)

        sys.stderr.write(err)



        assert str(sys_exit.value) == '0'

        assert return_value == 0

        if expected_none:

            assert re.search(expected[0], err) is None, f"Not expected string in log: {expected[0]}"

            assert re.search(expected[1], err) is None, f"Not expected: {expected[1]}"
@@ -117,13 +103,12 @@ class TestDisable: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

            pytest.raises(SystemExit) as sys_exit:

            self.loader.exec_module(self.twister_module)



        return_value = twister_main(args)



        out, err = capfd.readouterr()

        sys.stdout.write(out)

        sys.stderr.write(err)



        assert str(sys_exit.value) == expected_exit_code, \

            f"Twister return not expected ({expected_exit_code}) exit code: ({sys_exit.value})"
 
        assert return_value == expected_exit_code, \

            f"Twister return not expected ({expected_exit_code}) exit code: ({return_value})"

scripts/tests/twister_blackbox/test_testlist.py

+4 −24
Original line number Diff line number Diff line
@@ -6,29 +6,17 @@ 
Blackbox tests for twister's command line functions related to saving and loading a testlist.

"""



import importlib

from unittest import mock

import os

import pytest

import sys

import json



# pylint: disable=no-name-in-module

from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock, clear_log_in_test

from conftest import TEST_DATA, suite_filename_mock, clear_log_in_test

from twisterlib.testplan import TestPlan

from twisterlib.twister_main import main as twister_main





class TestTestlist:

    @classmethod

    def setup_class(cls):

        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')

        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)

        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)

        cls.twister_module = importlib.util.module_from_spec(cls.spec)



    @classmethod

    def teardown_class(cls):

        pass



    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)

    def test_save_tests(self, out_path):
@@ -42,11 +30,7 @@ class TestTestlist: 
               ) for val in pair]



        # Save agnostics tests

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(SystemExit) as sys_exit:

            self.loader.exec_module(self.twister_module)



        assert str(sys_exit.value) == '0'

        assert twister_main(args) == 0



        clear_log_in_test()
@@ -58,11 +42,7 @@ class TestTestlist: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(SystemExit) as sys_exit:

            self.loader.exec_module(self.twister_module)



        assert str(sys_exit.value) == '0'

        assert twister_main(args) == 0



        with open(os.path.join(out_path, 'testplan.json')) as f:

            j = json.load(f)

scripts/tests/twister_blackbox/test_testplan.py

+20 −42
Original line number Diff line number Diff line
@@ -6,22 +6,21 @@ 
Blackbox tests for twister's command line functions - those requiring testplan.json

"""



import importlib

from unittest import mock

import os

import pytest

import sys

import json



# pylint: disable=no-name-in-module

from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock

from conftest import TEST_DATA, suite_filename_mock

from twisterlib.testplan import TestPlan

from twisterlib.error import TwisterRuntimeError

from twisterlib.twister_main import main as twister_main





class TestTestPlan:

    TESTDATA_1 = [

        ('dummy.agnostic.group2.a2_tests.assert1', SystemExit, 4),

        ('dummy.agnostic.group2.a2_tests.assert1', None, 4),

        (

            os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests',

                         'dummy', 'agnostic', 'group1', 'subgroup1',
@@ -39,17 +38,6 @@ class TestTestPlan: 
        (False, 7),

    ]



    @classmethod

    def setup_class(cls):

        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')

        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)

        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)

        cls.twister_module = importlib.util.module_from_spec(cls.spec)



    @classmethod

    def teardown_class(cls):

        pass



    @pytest.mark.parametrize(

        'test, expected_exception, expected_subtest_count',

        TESTDATA_1,
@@ -65,14 +53,11 @@ class TestTestPlan: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(expected_exception) as exc:

            self.loader.exec_module(self.twister_module)



        if expected_exception != SystemExit:

            assert True

            return



        if expected_exception:

            with pytest.raises(expected_exception):

                twister_main(args)

        else:

            return_value = twister_main(args)

            with open(os.path.join(out_path, 'testplan.json')) as f:

                j = json.load(f)

            filtered_j = [
@@ -81,7 +66,7 @@ class TestTestPlan: 
                    for tc in ts['testcases'] if 'reason' not in tc

            ]



        assert str(exc.value) == '0'

            assert return_value == 0

            assert len(filtered_j) == expected_subtest_count



    @pytest.mark.parametrize(
@@ -98,11 +83,8 @@ class TestTestPlan: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(SystemExit) as exc:

            self.loader.exec_module(self.twister_module)

        assert twister_main(args)  == 0



        assert str(exc.value) == '0'

        import pprint

        with open(os.path.join(out_path, 'testplan.json')) as f:

            j = json.load(f)
@@ -132,11 +114,7 @@ class TestTestPlan: 
                   ['-p'] * len(test_platforms), test_platforms

               ) for val in pair]



        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \

                pytest.raises(SystemExit) as exc:

            self.loader.exec_module(self.twister_module)



        assert str(exc.value) == '0'

        assert twister_main(args) == 0



        with open(os.path.join(out_path, 'testplan.json')) as f:

            j = json.load(f)

CRA Git | Maintained and supported by SUSTech CRA and CCSE