408 lines
18 KiB
Python
408 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''
|
|
This test file contains testsuites for Testsuite class of twister
|
|
'''
|
|
import sys
|
|
import os
|
|
import pytest
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
|
|
|
from twisterlib.testplan import TestPlan
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.testsuite import TestSuite
|
|
from twisterlib.platform import Platform
|
|
from twisterlib.quarantine import Quarantine
|
|
|
|
|
|
def test_testplan_add_testsuites(class_testplan):
|
|
""" Testing add_testcase function of Testsuite class in twister """
|
|
# Test 1: Check the list of testsuites after calling add testsuites function is as expected
|
|
class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml'
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/'
|
|
expected_testsuites = ['test_b.check_1',
|
|
'test_b.check_2',
|
|
'test_c.check_1',
|
|
'test_c.check_2',
|
|
'test_a.check_1',
|
|
'test_a.check_2',
|
|
'test_d.check_1',
|
|
'test_e.check_1',
|
|
'sample_test.app',
|
|
'test_config.main']
|
|
testsuite_list = []
|
|
for key in sorted(class_testplan.testsuites.keys()):
|
|
testsuite_list.append(os.path.basename(os.path.normpath(key)))
|
|
assert sorted(testsuite_list) == sorted(expected_testsuites)
|
|
|
|
# Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects
|
|
suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1')
|
|
assert suite.name == tests_rel_dir + 'test_a/test_a.check_1'
|
|
assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values())
|
|
|
|
@pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")])
|
|
def test_add_configurations(test_data, class_env, board_root_dir):
|
|
""" Testing add_configurations function of TestPlan class in Twister
|
|
Test : Asserting on default platforms list
|
|
"""
|
|
class_env.board_roots = [os.path.abspath(test_data + board_root_dir)]
|
|
plan = TestPlan(class_env)
|
|
plan.parse_configuration(config_file=class_env.test_config)
|
|
if board_root_dir == "board_config":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) == sorted(['demo_board_1', 'demo_board_3'])
|
|
elif board_root_dir == "board_config_file_not_exist":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) != sorted(['demo_board_1'])
|
|
|
|
|
|
def test_get_all_testsuites(class_testplan, all_testsuites_dict):
|
|
""" Testing get_all_testsuites function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.testsuites = all_testsuites_dict
|
|
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
|
|
'test_a.check_1.1c',
|
|
'test_a.check_1.2a', 'test_a.check_1.2b',
|
|
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
|
|
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
|
|
'test_a.check_2.1c', 'test_a.check_2.2a',
|
|
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
|
|
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
|
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
|
'test_c.check_2', 'test_d.check_1.unit_1a',
|
|
'test_d.check_1.unit_1b',
|
|
'test_e.check_1.1a', 'test_e.check_1.1b',
|
|
'test_config.main']
|
|
print(sorted(plan.get_all_tests()))
|
|
print(sorted(expected_tests))
|
|
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
|
|
|
|
def test_get_platforms(class_testplan, platforms_list):
|
|
""" Testing get_platforms function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_1")
|
|
assert isinstance(platform, Platform)
|
|
assert platform.name == "demo_board_1"
|
|
|
|
TESTDATA_PART1 = [
|
|
("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"),
|
|
("platform_allow", ['demo_board_1'], None, None, "Not in testsuite platform allow list"),
|
|
("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"),
|
|
("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"),
|
|
("arch_exclude", ['x86_demo'], None, None, "In test case arch exclude"),
|
|
("arch_allow", ['arm'], None, None, "Not in test case arch allow list"),
|
|
("skip", True, None, None, "Skip filter"),
|
|
("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"),
|
|
("min_flash", "2024", "flash", "1024", "Not enough FLASH"),
|
|
("min_ram", "500", "ram", "256", "Not enough RAM"),
|
|
("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"),
|
|
("build_on_all", True, None, None, "Platform is excluded on command line."),
|
|
(None, None, "supported_toolchains", ['gcc'], "Not supported by the toolchain"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards",
|
|
TESTDATA_PART1)
|
|
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 1: Response of apply_filters function have
|
|
appropriate values according to the filters
|
|
"""
|
|
plan = class_testplan
|
|
if tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = all_testsuites_dict
|
|
for plat in plan.platforms:
|
|
if plat_attribute == "ignore_tags":
|
|
plat.ignore_tags = plat_value
|
|
if plat_attribute == "flash":
|
|
plat.flash = plat_value
|
|
if plat_attribute == "ram":
|
|
plat.ram = plat_value
|
|
if plat_attribute == "env":
|
|
plat.env = plat_value
|
|
plat.env_satisfied = False
|
|
if plat_attribute == "supported_toolchains":
|
|
plat.supported_toolchains = plat_value
|
|
for _, testcase in plan.testsuites.items():
|
|
if tc_attribute == "toolchain_allow":
|
|
testcase.toolchain_allow = tc_value
|
|
if tc_attribute == "platform_allow":
|
|
testcase.platform_allow = tc_value
|
|
if tc_attribute == "toolchain_exclude":
|
|
testcase.toolchain_exclude = tc_value
|
|
if tc_attribute == "platform_exclude":
|
|
testcase.platform_exclude = tc_value
|
|
if tc_attribute == "arch_exclude":
|
|
testcase.arch_exclude = tc_value
|
|
if tc_attribute == "arch_allow":
|
|
testcase.arch_allow = tc_value
|
|
if tc_attribute == "skip":
|
|
testcase.skip = tc_value
|
|
if tc_attribute == "tags":
|
|
testcase.tags = tc_value
|
|
if tc_attribute == "min_flash":
|
|
testcase.min_flash = tc_value
|
|
if tc_attribute == "min_ram":
|
|
testcase.min_ram = tc_value
|
|
|
|
if tc_attribute == "build_on_all":
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.build_on_all = tc_value
|
|
plan.apply_filters(exclude_platform=['demo_board_1'])
|
|
elif plat_attribute == "supported_toolchains":
|
|
plan.apply_filters(force_toolchain=False,
|
|
exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
elif tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
else:
|
|
plan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
TESTDATA_PART2 = [
|
|
("runnable", "True", "Not runnable on device"),
|
|
("exclude_tag", ['test_a'], "Command line testsuite exclude filter"),
|
|
("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"),
|
|
("arch", ['arm_test'], "Command line testsuite arch filter"),
|
|
("tag", ['test_d'], "Command line testsuite tag filter")
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2)
|
|
def test_apply_filters_part2(class_testplan, all_testsuites_dict,
|
|
platforms_list, extra_filter, extra_filter_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 2 : Response of apply_filters function (discard dictionary) have
|
|
appropriate values according to the filters
|
|
"""
|
|
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
kwargs = {
|
|
extra_filter : extra_filter_value,
|
|
"exclude_platform" : [
|
|
'demo_board_1'
|
|
],
|
|
"platform" : [
|
|
'demo_board_2'
|
|
]
|
|
}
|
|
class_testplan.apply_filters(**kwargs)
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
|
|
TESTDATA_PART3 = [
|
|
(20, 20, -1, 0),
|
|
(-2, -1, 10, 20),
|
|
(0, 0, 0, 0)
|
|
]
|
|
|
|
@pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram",
|
|
TESTDATA_PART3)
|
|
def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_min_flash, plat_flash, tc_min_ram, plat_ram):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 3 : Testing edge cases for ram and flash values of platforms & testsuites
|
|
"""
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
|
|
for plat in class_testplan.platforms:
|
|
plat.flash = plat_flash
|
|
plat.ram = plat_ram
|
|
for _, testcase in class_testplan.testsuites.items():
|
|
testcase.min_ram = tc_min_ram
|
|
testcase.min_flash = tc_min_flash
|
|
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
assert not filtered_instances
|
|
|
|
def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list):
|
|
""" Testing add_instances() function of TestPlan class in Twister
|
|
Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name)
|
|
Test 2: Values of 'instances' dictionary in Testsuite class are an
|
|
instance of 'TestInstance' class
|
|
Test 3: Values of 'instances' dictionary have expected values.
|
|
"""
|
|
class_env.outdir = test_data
|
|
plan = TestPlan(class_env)
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_2")
|
|
instance_list = []
|
|
for _, testcase in all_testsuites_dict.items():
|
|
instance = TestInstance(testcase, platform, class_env.outdir)
|
|
instance_list.append(instance)
|
|
plan.add_instances(instance_list)
|
|
assert list(plan.instances.keys()) == \
|
|
[platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
|
|
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
|
|
assert list(plan.instances.values()) == instance_list
|
|
|
|
|
|
QUARANTINE_BASIC = {
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
|
|
}
|
|
|
|
QUARANTINE_WITH_REGEXP = {
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
|
|
}
|
|
|
|
QUARANTINE_PLATFORM = {
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_e/test_e.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
|
|
}
|
|
|
|
QUARANTINE_MULTIFILES = {
|
|
**QUARANTINE_BASIC,
|
|
**QUARANTINE_WITH_REGEXP
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("quarantine_files, quarantine_verify, expected_val"),
|
|
[
|
|
(['basic.yaml'], False, QUARANTINE_BASIC),
|
|
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
|
|
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
|
|
(['platform.yaml'], False, QUARANTINE_PLATFORM),
|
|
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
|
|
(['empty.yaml'], False, {})
|
|
],
|
|
ids=[
|
|
'basic',
|
|
'with_regexp',
|
|
'quarantine_verify',
|
|
'platform',
|
|
'multifiles',
|
|
'empty'
|
|
])
|
|
def test_quarantine(class_testplan, platforms_list, test_data,
|
|
quarantine_files, quarantine_verify, expected_val):
|
|
""" Testing quarantine feature in Twister
|
|
"""
|
|
class_testplan.options.all = True
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
quarantine_list = [
|
|
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
|
|
]
|
|
class_testplan.quarantine = Quarantine(quarantine_list)
|
|
class_testplan.options.quarantine_verify = quarantine_verify
|
|
class_testplan.apply_filters()
|
|
|
|
for testname, instance in class_testplan.instances.items():
|
|
if quarantine_verify:
|
|
if testname in expected_val:
|
|
assert not instance.status
|
|
else:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Not under quarantine"
|
|
else:
|
|
print(testname)
|
|
if testname in expected_val:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Quarantine: " + expected_val[testname]
|
|
else:
|
|
assert not instance.status
|
|
|
|
def test_required_snippets_app(class_testplan, all_testsuites_dict, platforms_list):
|
|
""" Testing required_snippets function of TestPlan class in Twister
|
|
Ensure that app snippets work and are only applied to boards that support the snippet
|
|
"""
|
|
plan = class_testplan
|
|
testsuite = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1')
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = {'scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1': testsuite}
|
|
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.exclude_platform = []
|
|
testcase.required_snippets = ['dummy']
|
|
testcase.build_on_all = True
|
|
|
|
plan.apply_filters()
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == "Snippet not supported"
|
|
|
|
def test_required_snippets_global(class_testplan, all_testsuites_dict, platforms_list):
|
|
""" Testing required_snippets function of TestPlan class in Twister
|
|
Ensure that global snippets work and application does not fail
|
|
"""
|
|
plan = class_testplan
|
|
testsuite = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1')
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = {'scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1': testsuite}
|
|
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.exclude_platform = []
|
|
testcase.required_snippets = ['cdc-acm-console']
|
|
testcase.build_on_all = True
|
|
|
|
plan.apply_filters()
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
assert len(filtered_instances) == 0
|
|
|
|
def test_required_snippets_multiple(class_testplan, all_testsuites_dict, platforms_list):
|
|
""" Testing required_snippets function of TestPlan class in Twister
|
|
Ensure that multiple snippets can be used and are applied
|
|
"""
|
|
plan = class_testplan
|
|
testsuite = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1')
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = {'scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1': testsuite}
|
|
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.exclude_platform = []
|
|
testcase.required_snippets = ['dummy', 'cdc-acm-console']
|
|
testcase.build_on_all = True
|
|
|
|
plan.apply_filters()
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
assert len(filtered_instances) == 2
|
|
for d in filtered_instances:
|
|
assert d.reason == "Snippet not supported"
|