testcases: add templates for cts and vts tests

which could be used by default to run the full cts vts package,
And these files are generic files, will be reused
by other projects via the extentds

Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

lava_test_plans/projects/lkft-android/variables.ini

@@ -34,3 +34,9 @@ DEPLOY_OS="android"
 
 #
 DOCKER_IMAGE_TEST="docker-hub-url"
+
+#
+TEST_CTS_URL="test-cts-url"
+TEST_VTS_URL="test-vts-url"
+TEST_CTS_VERSION="test-cts-version"
+TEST_VTS_VERSION="test-vts-version"

lava_test_plans/testcases/android-cts.yaml

@@ -0,0 +1,12 @@
+{% extends "testcases/master/template-android-xts.jinja2" %}
+
+{% set xts_test_url = TEST_CTS_URL %}
+{% set xts_version = TEST_CTS_VERSION %}
+
+{% set xts_test_params = xts_test_params|default("cts") %}
+{% set test_name = test_name|default("cts") %}
+{% set xts_test_suite_name = xts_test_suite_name|default("cts-lkft") %}
+{% set xts_test_path = xts_test_path|default("android-cts") %}
+{% set xts_pkg_name = xts_pkg_name|default("android-cts.zip") %}
+{% set xts_expect_reboot = xts_expect_reboot|default(false) %}
+{% set xts_metadata_prefix = xts_metadata_prefix|default("cts") %}

lava_test_plans/testcases/android-vts.yaml

@@ -0,0 +1,12 @@
+{% extends "testcases/master/template-android-xts.jinja2" %}
+
+{% set xts_test_url = TEST_VTS_URL %}
+{% set xts_version = TEST_VTS_VERSION %}
+
+{% set xts_test_params = xts_test_params|default("vts") %}
+{% set test_name = test_name|default("vts") %}
+{% set xts_test_suite_name = xts_test_suite_name|default("vts-test") %}
+{% set xts_test_path = xts_test_path|default("android-vts") %}
+{% set xts_pkg_name = xts_pkg_name|default("android-vts.zip") %}
+{% set xts_expect_reboot = xts_expect_reboot|default(true) %}
+{% set xts_metadata_prefix = xts_metadata_prefix|default("vts") %}

lava_test_plans/testcases/master/template-android-xts.jinja2

@@ -0,0 +1,38 @@
+{% extends "testcases/master/template-master.jinja2" %}
+
+{% set xts_test_params = xts_test_params|default("xts-not-work") %}
+{## the test_name is a part of the job name ##}
+{% set test_name = test_name|default("xts") %}
+{## the test_suite_name is a part of the path of the lava result ##}
+{% set xts_test_suite_name = xts_test_suite_name|default(test_name) %}
+{% set xts_test_path = xts_test_path|default("xts-path") %}
+{% set xts_pkg_name = xts_pkg_name|default("android-xts.zip") %}
+{% set xts_expect_reboot = xts_expect_reboot|default(false) %}
+{% set xts_result_format = xts_result_format|default("aggregated") %}
+{% set xts_test_definition_path = xts_test_definition_path|default("automated/android/noninteractive-tradefed/tradefed.yaml") %}
+{% set xts_metadata_prefix = xts_metadata_prefix|default('xts') %}
+{% set xts_version = xts_version|default("xts-version-unknown") %}
+
+{% block metadata %}
+  {{ super() }}
+  {{xts_metadata_prefix}}-url: "{{ xts_test_url }}/{{xts_pkg_name}}"
+  {{xts_metadata_prefix}}-manifest: "{{ xts_test_url }}/pinned-manifest.xml"
+  {{xts_metadata_prefix}}-version: "{{ xts_version }}"
+{% endblock metadata %}
+
+{% block test_target %}
+  {{ super() }}
+    - repository: {{TEST_DEFINITIONS_REPOSITORY}}
+      from: git
+      path: {{xts_test_definition_path}}
+      params:
+        TEST_PARAMS: {% block xts_test_params %}{{xts_test_params}}{% endblock xts_test_params %}
+        TEST_URL: "{{xts_test_url}}/{{xts_pkg_name}}"
+        TEST_PATH: "{{xts_test_path}}"
+        RESULTS_FORMAT: "{{xts_result_format}}"
+        ANDROID_VERSION: "{{ANDROID_VERSION}}"
+{% if xts_expect_reboot %}
+        TEST_REBOOT_EXPECTED: "true"
+{% endif %}
+      name: "{{xts_test_suite_name}}"
+{% endblock test_target %}

test/test_lava_test_plans.py

@@ -9,9 +9,9 @@
 
 test_lava_validity = "" if os.getenv("SKIP_TEST_LAVA_VALIDITY") else "--test-lava-validity"
 
-# all tests all devices
+# all Linux tests all devices
 devices = [os.path.basename(d) for d in glob.glob("lava_test_plans/devices/*")]
-testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/*.yaml")]
+testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/[!android-]*.yaml")]
 variable_input_file = "variables.ini"
 tests = []
 for device in devices:
@@ -127,21 +127,40 @@ def test_call_lava_test_plan_testplans_ti(param):
 lkft_android_project_device_path = "lava_test_plans/projects/lkft-android/devices"
 lkft_android_devices = [os.path.basename(d) for d in glob.glob("lava_test_plans/projects/lkft-android/devices/*")]
 assert len(lkft_android_devices) > 0
-lkft_android_testcases = [ "boot.yaml" ]
-assert len(lkft_android_testcases) > 0
+lkft_android_boot_testcases = [ "boot.yaml" ]
+assert len(lkft_android_boot_testcases) > 0
+lkft_android_default_variables_file = "lava_test_plans/projects/lkft-android/variables.ini"
 lkft_android_variable_input_files = [
-                         "lava_test_plans/projects/lkft-android/variables.ini",
+                         lkft_android_default_variables_file,
                          "test/variables-lkft-android-tuxsuite.ini",
                        ]
-assert len(lkft_android_testcases) > 0
 tests = []
 for device in lkft_android_devices:
-    for testcase in lkft_android_testcases:
+    for testcase in lkft_android_boot_testcases:
         for variable_input_file in lkft_android_variable_input_files:
-            tests.append((variable_input_file, device, testcase, lkft_android_project_device_path))
+            tests.append((variable_input_file,
+                          device,
+                          testcase,
+                          lkft_android_project_device_path))
+
+lkft_android_nonboot_testcases = [os.path.basename(d) for d in glob.glob("lava_test_plans/testcases/android-*.yaml")]
+assert len(lkft_android_nonboot_testcases) > 0
+for device in lkft_android_devices:
+    for testcase in lkft_android_nonboot_testcases:
+        tests.append((lkft_android_default_variables_file,
+                      device,
+                      testcase,
+                      lkft_android_project_device_path))
 
 @pytest.mark.parametrize("param", tests)
 def test_call_lava_test_plan_testcase_lkft_android(param):
     variable_input_file, device, testcase, project_device_path = param
-    sys.argv = shlex.split(f'lava_test_plans --dry-run --variables "{variable_input_file}"  --testplan-device-path "{project_device_path}" --device-type "{device}" --test-case "{testcase}" {test_lava_validity}')
+    sys.argv = shlex.split(
+                    f'''lava_test_plans
+                            --dry-run
+                            --variables "{variable_input_file}"
+                            --testplan-device-path "{project_device_path}"
+                            --device-type "{device}"
+                            --test-case "{testcase}"
+                            {test_lava_validity}''')
     assert main() == 0, f"fail: {sys.argv}"