Add support for additional instance types to integration tests

This commit allows integration tests configuration files to use instance types provided with the `additional_instance_types` parameter to the `test_runner.py` script.
Signed-off-by: ddeidda <ddeidda@amazon.com>

Author: ddeidda

tests/integration-tests/configs/additional_instance_types.yaml

@@ -0,0 +1,10 @@
+{%- import 'common.jinja2' as common with context -%}
+---
+test-suites:
+  schedulers:
+    test_slurm.py::test_slurm:
+      dimensions:
+        - regions: ["us-east-2"]
+          instances: [{{ common.instance("instance_type_1") }}]
+          oss: {{ common.OSS_COMMERCIAL_X86 }}
+          schedulers: ["slurm"]

tests/integration-tests/configs/common.jinja2

@@ -15,3 +15,11 @@
 {%- set INSTANCES_DEFAULT_X86 = ["c5.xlarge"] -%}
 {%- set INSTANCES_DEFAULT_ARM = ["m6g.xlarge"] -%}
 {%- set INSTANCES_DEFAULT = ["c5.xlarge", "m6g.xlarge"] -%}
+
+{%- macro instance(instance_key) -%}
+    {%- if additional_instance_types_map -%}
+        {{ additional_instance_types_map.get(instance_key, instance_key) }}
+    {%- else -%}
+        "{{ instance_key }}"
+    {%- endif -%}
+{%- endmacro -%}

tests/integration-tests/conftest.py

@@ -126,9 +126,9 @@ def pytest_configure(config):
 
     # Read instance types data file if used
     if config.getoption("instance_types_data_file", None):
-        config.option.instance_types_data = _read_json_file(config.getoption("instance_types_data_file"))
         # Load additional instance types data
-        _set_additional_instance_types_data(config.option.instance_types_data)
+        InstanceTypesData.load_additional_instance_types_data(config.getoption("instance_types_data_file"))
+        config.option.instance_types_data = InstanceTypesData.additional_instance_types_data
 
     # register additional markers
     config.addinivalue_line("markers", "instances(instances_list): run test only against the listed instances.")
@@ -207,11 +207,6 @@ def _log_collected_tests(session):
             out_f.write("\n")
 
 
-def _set_additional_instance_types_data(instance_types_data):
-    InstanceTypesData.additional_instance_types_data = instance_types_data
-    logging.info("Additional instance types data loaded: {0}".format(InstanceTypesData.additional_instance_types_data))
-
-
 def pytest_exception_interact(node, call, report):
     """Called when an exception was raised which can potentially be interactively handled.."""
     logging.error(
@@ -870,12 +865,3 @@ def _pcluster_ami_without_standard_naming(version):
     if ami_id:
         client = boto3.client("ec2", region_name=region)
         client.deregister_image(ImageId=ami_id)
-
-
-def _read_json_file(file):
-    """Read a Json file into a String and raise an exception if the file is invalid."""
-    try:
-        with open(file) as f:
-            return json.load(f)
-    except Exception:
-        logging.exception("Failed when reading json file %s", file)

tests/integration-tests/framework/tests_configuration/config_renderer.py

@@ -15,6 +15,7 @@
 
 import yaml
 from jinja2 import Environment, FileSystemLoader
+from utils import InstanceTypesData
 
 
 def read_config_file(config_file, print_rendered=False):
@@ -56,7 +57,11 @@ def _render_config_file(config_file):
         config_dir = os.path.dirname(config_file)
         config_name = os.path.basename(config_file)
         file_loader = FileSystemLoader(config_dir)
-        return Environment(loader=file_loader).get_template(config_name).render()
+        return (
+            Environment(loader=file_loader)
+            .get_template(config_name)
+            .render(additional_instance_types_map=InstanceTypesData.additional_instance_types_map)
+        )
     except Exception:
         logging.error("Failed when rendering config file %s", config_file)
         raise

tests/integration-tests/test_runner.py

@@ -25,6 +25,7 @@
 from framework.tests_configuration.config_utils import get_all_regions
 from framework.tests_configuration.config_validator import assert_valid_config
 from reports_generator import generate_cw_report, generate_json_report, generate_junitxml_merged_report
+from utils import InstanceTypesData
 
 logger = logging.getLogger()
 logging.basicConfig(format="%(asctime)s - %(levelname)s - %(module)s - %(message)s", level=logging.INFO)
@@ -122,7 +123,6 @@ def _init_argparser():
         "Note that when a config file is used the following flags are ignored: instances, regions, oss, schedulers. "
         "Refer to the docs for further details on the config format: "
         "https://github.com/aws/aws-parallelcluster/blob/develop/tests/integration-tests/README.md",
-        type=_test_config_file,
     )
     dimensions_group.add_argument(
         "-i",
@@ -559,6 +559,7 @@ def _check_args(args):
         assert_that(args.schedulers).described_as("--schedulers cannot be empty").is_not_empty()
     else:
         try:
+            args.tests_config = _test_config_file(args.tests_config)
             assert_valid_config(args.tests_config, args.tests_root_dir)
             logger.info("Found valid config file:\n%s", dump_rendered_config_file(args.tests_config))
         except Exception:
@@ -582,6 +583,12 @@ def main():
         exit(1)
 
     args = _init_argparser().parse_args()
+
+    # Load additional instance types data, if provided.
+    # This step must be done before loading test config files in order to resolve instance type placeholders.
+    if args.instance_types_data:
+        InstanceTypesData.load_additional_instance_types_data(args.instance_types_data)
+
     _check_args(args)
     logger.info("Parsed test_runner parameters {0}".format(args))
 

tests/integration-tests/utils.py

@@ -9,6 +9,7 @@
 # or in the "LICENSE.txt" file accompanying this file.
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import json
 import logging
 import os
 import random
@@ -28,6 +29,25 @@ class InstanceTypesData:
 
     # Additional instance types data provided via tests configuration
     additional_instance_types_data = {}
+    additional_instance_types_map = {}
+
+    @staticmethod
+    def load_additional_instance_types_data(instance_types_data_file):
+        """
+        Load additional instance types data from configuration json file.
+        The file must contain two keys:
+          - "instance_types_data": The json structure to be passed to cluster configuration files
+          - "instance_types_map": A dict containing logical instance types names (that can be used in cluster config
+            files) vs real ones (e.g. "t2.micro")
+        """
+        instance_types_data_file_content = read_json_file(instance_types_data_file)
+        InstanceTypesData.additional_instance_types_data = instance_types_data_file_content.get(
+            "instance_types_data", {}
+        )
+        InstanceTypesData.additional_instance_types_map = instance_types_data_file_content.get("instance_types_map", {})
+        logging.info(
+            "Additional instance types data loaded: {0}".format(InstanceTypesData.additional_instance_types_data)
+        )
 
     @staticmethod
     def get_instance_info(instance_type, region_name=None):
@@ -412,3 +432,13 @@ def check_headnode_security_group(region, cluster, port, expected_cidr):
     ips = response["SecurityGroups"][0]["IpPermissions"]
     target = next(filter(lambda x: x.get("FromPort", -1) == port, ips), {})
     assert_that(target["IpRanges"][0]["CidrIp"]).is_equal_to(expected_cidr)
+
+
+def read_json_file(file):
+    """Read a Json file into a String and raise an exception if the file is invalid."""
+    try:
+        with open(file) as f:
+            return json.load(f)
+    except Exception as e:
+        logging.exception("Failed when reading json file %s", file)
+        raise e