From 9cf2fac905a1236796ed2cfe45de1750e73904b1 Mon Sep 17 00:00:00 2001 From: Yongqin Liu Date: Fri, 31 Mar 2023 23:59:56 +0800 Subject: [PATCH] testcases: add templates for cts and vts tests which could be used by default to run the full cts vts package, And these files are generic files, will be reused by other projects via the extentds Signed-off-by: Yongqin Liu --- .../projects/lkft-android/variables.ini | 6 +++ lava_test_plans/testcases/android-cts.yaml | 12 ++++++ lava_test_plans/testcases/android-vts.yaml | 12 ++++++ .../master/template-android-xts.jinja2 | 38 +++++++++++++++++++ test/test_lava_test_plans.py | 37 +++++++++++++----- 5 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 lava_test_plans/testcases/android-cts.yaml create mode 100644 lava_test_plans/testcases/android-vts.yaml create mode 100644 lava_test_plans/testcases/master/template-android-xts.jinja2 diff --git a/lava_test_plans/projects/lkft-android/variables.ini b/lava_test_plans/projects/lkft-android/variables.ini index 3ceb9eeb..6dc3af24 100644 --- a/lava_test_plans/projects/lkft-android/variables.ini +++ b/lava_test_plans/projects/lkft-android/variables.ini @@ -34,3 +34,9 @@ DEPLOY_OS="android" # DOCKER_IMAGE_TEST="docker-hub-url" + +# +TEST_CTS_URL="test-cts-url" +TEST_VTS_URL="test-vts-url" +TEST_CTS_VERSION="test-cts-version" +TEST_VTS_VERSION="test-vts-version" diff --git a/lava_test_plans/testcases/android-cts.yaml b/lava_test_plans/testcases/android-cts.yaml new file mode 100644 index 00000000..7443e954 --- /dev/null +++ b/lava_test_plans/testcases/android-cts.yaml @@ -0,0 +1,12 @@ +{% extends "testcases/master/template-android-xts.jinja2" %} + +{% set xts_test_url = TEST_CTS_URL %} +{% set xts_version = TEST_CTS_VERSION %} + +{% set xts_test_params = xts_test_params|default("cts") %} +{% set test_name = test_name|default("cts") %} +{% set xts_test_suite_name = xts_test_suite_name|default("cts-lkft") %} +{% set xts_test_path = xts_test_path|default("android-cts") %} +{% set xts_pkg_name = xts_pkg_name|default("android-cts.zip") %} +{% set xts_expect_reboot = xts_expect_reboot|default(false) %} +{% set xts_metadata_prefix = xts_metadata_prefix|default("cts") %} diff --git a/lava_test_plans/testcases/android-vts.yaml b/lava_test_plans/testcases/android-vts.yaml new file mode 100644 index 00000000..f0d162ba --- /dev/null +++ b/lava_test_plans/testcases/android-vts.yaml @@ -0,0 +1,12 @@ +{% extends "testcases/master/template-android-xts.jinja2" %} + +{% set xts_test_url = TEST_VTS_URL %} +{% set xts_version = TEST_VTS_VERSION %} + +{% set xts_test_params = xts_test_params|default("vts") %} +{% set test_name = test_name|default("vts") %} +{% set xts_test_suite_name = xts_test_suite_name|default("vts-test") %} +{% set xts_test_path = xts_test_path|default("android-vts") %} +{% set xts_pkg_name = xts_pkg_name|default("android-vts.zip") %} +{% set xts_expect_reboot = xts_expect_reboot|default(true) %} +{% set xts_metadata_prefix = xts_metadata_prefix|default("vts") %} diff --git a/lava_test_plans/testcases/master/template-android-xts.jinja2 b/lava_test_plans/testcases/master/template-android-xts.jinja2 new file mode 100644 index 00000000..f7da8a69 --- /dev/null +++ b/lava_test_plans/testcases/master/template-android-xts.jinja2 @@ -0,0 +1,38 @@ +{% extends "testcases/master/template-master.jinja2" %} + +{% set xts_test_params = xts_test_params|default("xts-not-work") %} +{## the test_name is a part of the job name ##} +{% set test_name = test_name|default("xts") %} +{## the test_suite_name is a part of the path of the lava result ##} +{% set xts_test_suite_name = xts_test_suite_name|default(test_name) %} +{% set xts_test_path = xts_test_path|default("xts-path") %} +{% set xts_pkg_name = xts_pkg_name|default("android-xts.zip") %} +{% set xts_expect_reboot = xts_expect_reboot|default(false) %} +{% set xts_result_format = xts_result_format|default("aggregated") %} +{% set xts_test_definition_path = xts_test_definition_path|default("automated/android/noninteractive-tradefed/tradefed.yaml") %} +{% set xts_metadata_prefix = xts_metadata_prefix|default('xts') %} +{% set xts_version = xts_version|default("xts-version-unknown") %} + +{% block metadata %} + {{ super() }} + {{xts_metadata_prefix}}-url: "{{ xts_test_url }}/{{xts_pkg_name}}" + {{xts_metadata_prefix}}-manifest: "{{ xts_test_url }}/pinned-manifest.xml" + {{xts_metadata_prefix}}-version: "{{ xts_version }}" +{% endblock metadata %} + +{% block test_target %} + {{ super() }} + - repository: {{TEST_DEFINITIONS_REPOSITORY}} + from: git + path: {{xts_test_definition_path}} + params: + TEST_PARAMS: {% block xts_test_params %}{{xts_test_params}}{% endblock xts_test_params %} + TEST_URL: "{{xts_test_url}}/{{xts_pkg_name}}" + TEST_PATH: "{{xts_test_path}}" + RESULTS_FORMAT: "{{xts_result_format}}" + ANDROID_VERSION: "{{ANDROID_VERSION}}" +{% if xts_expect_reboot %} + TEST_REBOOT_EXPECTED: "true" +{% endif %} + name: "{{xts_test_suite_name}}" +{% endblock test_target %} diff --git a/test/test_lava_test_plans.py b/test/test_lava_test_plans.py index b86b9526..abd59e6c 100644 --- a/test/test_lava_test_plans.py +++ b/test/test_lava_test_plans.py @@ -9,9 +9,9 @@ test_lava_validity = "" if os.getenv("SKIP_TEST_LAVA_VALIDITY") else "--test-lava-validity" -# all tests all devices +# all Linux tests all devices devices = [os.path.basename(d) for d in glob.glob("lava_test_plans/devices/*")] -testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/*.yaml")] +testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/[!android-]*.yaml")] variable_input_file = "variables.ini" tests = [] for device in devices: @@ -127,21 +127,40 @@ def test_call_lava_test_plan_testplans_ti(param): lkft_android_project_device_path = "lava_test_plans/projects/lkft-android/devices" lkft_android_devices = [os.path.basename(d) for d in glob.glob("lava_test_plans/projects/lkft-android/devices/*")] assert len(lkft_android_devices) > 0 -lkft_android_testcases = [ "boot.yaml" ] -assert len(lkft_android_testcases) > 0 +lkft_android_boot_testcases = [ "boot.yaml" ] +assert len(lkft_android_boot_testcases) > 0 +lkft_android_default_variables_file = "lava_test_plans/projects/lkft-android/variables.ini" lkft_android_variable_input_files = [ - "lava_test_plans/projects/lkft-android/variables.ini", + lkft_android_default_variables_file, "test/variables-lkft-android-tuxsuite.ini", ] -assert len(lkft_android_testcases) > 0 tests = [] for device in lkft_android_devices: - for testcase in lkft_android_testcases: + for testcase in lkft_android_boot_testcases: for variable_input_file in lkft_android_variable_input_files: - tests.append((variable_input_file, device, testcase, lkft_android_project_device_path)) + tests.append((variable_input_file, + device, + testcase, + lkft_android_project_device_path)) + +lkft_android_nonboot_testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/android-*.yaml")] +assert len(lkft_android_nonboot_testcases) > 0 +for device in lkft_android_devices: + for testcase in lkft_android_nonboot_testcases: + tests.append((lkft_android_default_variables_file, + device, + testcase, + lkft_android_project_device_path)) @pytest.mark.parametrize("param", tests) def test_call_lava_test_plan_testcase_lkft_android(param): variable_input_file, device, testcase, project_device_path = param - sys.argv = shlex.split(f'lava_test_plans --dry-run --variables "{variable_input_file}" --testplan-device-path "{project_device_path}" --device-type "{device}" --test-case "{testcase}" {test_lava_validity}') + sys.argv = shlex.split( + f'''lava_test_plans + --dry-run + --variables "{variable_input_file}" + --testplan-device-path "{project_device_path}" + --device-type "{device}" + --test-case "{testcase}" + {test_lava_validity}''') assert main() == 0, f"fail: {sys.argv}"