Allow passing arch-specific optarch on cmdline/config

Author: Flamefire

easybuild/tools/options.py

@@ -97,7 +97,8 @@
 from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
 from easybuild.tools.repository.repository import avail_repositories
-from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
+from easybuild.tools.systemtools import UNKNOWN, CPU_ARCHITECTURES, CPU_FAMILIES, CPU_VECTOR_EXTS
+from easybuild.tools.systemtools import check_python_version, get_cpu_architecture, get_cpu_family
 from easybuild.tools.systemtools import get_cpu_features, get_system_info
 from easybuild.tools.version import this_is_easybuild
 
@@ -930,29 +931,67 @@ def postprocess(self):
             cleanup_and_exit(self.tmpdir)
 
     def _postprocess_optarch(self):
-        """Postprocess --optarch option."""
-        optarch_parts = self.options.optarch.split(OPTARCH_SEP)
-
-        # we expect to find a ':' in every entry in optarch, in case optarch is specified on a per-compiler basis
-        n_parts = len(optarch_parts)
-        map_char_cnts = [p.count(OPTARCH_MAP_CHAR) for p in optarch_parts]
-        if (n_parts > 1 and any(c != 1 for c in map_char_cnts)) or (n_parts == 1 and map_char_cnts[0] > 1):
+        """
+        Postprocess --optarch option.
+
+        Format: map   := <entry>(;<space>*<entry>)*
+                entry := <compiler>(,<arch-spec)*:<flag-string>
+        Example values:
+           "GENERIC"
+           "march=native"
+           "Intel,x86:xHost; Intel,x86,AMD,AVX2:mavx2 -fma; GCC:march=native"
+        """
+        # Split into entries, and each entry into key-value pairs
+        optarch_parts = [i.strip().split(OPTARCH_MAP_CHAR) for i in self.options.optarch.split(OPTARCH_SEP)]
+
+        # We should either have a single value or a list of key-value pairs, and nothing else
+        is_single_value = len(optarch_parts) == 1 and len(optarch_parts[0]) == 1
+        if not is_single_value and any(len(i) != 2 for i in optarch_parts):
             raise EasyBuildError("The optarch option has an incorrect syntax: %s", self.options.optarch)
+
+        # if there are options for different compilers, we set up a dict
+        if is_single_value:
+            # if optarch is not in mapping format, we do nothing and just keep the string
+            self.log.info("Keeping optarch raw: %s", self.options.optarch)
         else:
-            # if there are options for different compilers, we set up a dict
-            if OPTARCH_MAP_CHAR in optarch_parts[0]:
-                optarch_dict = {}
-                for compiler, compiler_opt in [p.split(OPTARCH_MAP_CHAR) for p in optarch_parts]:
-                    if compiler in optarch_dict:
-                        raise EasyBuildError("The optarch option contains duplicated entries for compiler %s: %s",
-                                             compiler, self.options.optarch)
+            optarch_dict = {}
+            for key, compiler_opt in optarch_parts:
+                # key can be either a compiler (only) or compiler and archspec(s)
+                key_parts = key.split(',')
+                compiler = key_parts.pop(0)
+                if key_parts:
+                    for part, allowed_values in zip(key_parts, (CPU_ARCHITECTURES, CPU_FAMILIES, CPU_VECTOR_EXTS)):
+                        if part not in allowed_values:
+                            raise EasyBuildError("The optarch option has an incorrect syntax: %s\n"
+                                                 "'%s' of '%s' is not in allowed values: %s",
+                                                 self.options.optarch, part, key, allowed_values)
+                    arch_specs = tuple(key_parts) if len(key_parts) > 1 else key_parts[0]
+                else:
+                    arch_specs = None
+                try:
+                    compiler_dict = optarch_dict[compiler]
+                    # Convert single entry to dict if required
+                    if not isinstance(compiler_dict, dict):
+                        compiler_dict = {None: compiler_dict}
+                        optarch_dict[compiler] = compiler_dict
+                    if arch_specs in compiler_dict:
+                        if arch_specs is None:
+                            raise EasyBuildError("The optarch option contains a duplicated entry for compiler %s: %s",
+                                                 compiler, self.options.optarch)
+                        else:
+                            raise EasyBuildError("The optarch option contains a duplicated entry %s "
+                                                 "for compiler %s: %s",
+                                                 arch_specs, compiler, self.options.optarch)
                     else:
+                        compiler_dict[arch_specs] = compiler_opt
+                except KeyError:
+                    # Keep the dict flat when no archspecs are given
+                    if arch_specs is None:
                         optarch_dict[compiler] = compiler_opt
-                self.options.optarch = optarch_dict
-                self.log.info("Transforming optarch into a dict: %s", self.options.optarch)
-            # if optarch is not in mapping format, we do nothing and just keep the string
-            else:
-                self.log.info("Keeping optarch raw: %s", self.options.optarch)
+                    else:
+                        optarch_dict[compiler] = {arch_specs: compiler_opt}
+            self.options.optarch = optarch_dict
+            self.log.info("Transforming optarch into a dict: %s", self.options.optarch)
 
     def _postprocess_close_pr_reasons(self):
         """Postprocess --close-pr-reasons options"""

easybuild/tools/systemtools.py

@@ -1131,6 +1131,8 @@ def create_possible_keys():
         yield (arch, )
         # Also allow single string entry
         yield arch
+        # Default fallback for any arch
+        yield None
 
     result = None
     for key in create_possible_keys():

easybuild/tools/toolchain/compiler.py

@@ -333,7 +333,7 @@ def _set_optimal_architecture(self, default_optarch=None):
         :param default_optarch: default value to use for optarch, rather than using default value based on architecture
                                 (--optarch and --optarch=GENERIC still override this value)
         """
-        ec_optarch = self.options.get('optarch', False)
+        ec_optarch = self.options.get('optarch')
         if isinstance(ec_optarch, string_type):
             if OPTARCH_MAP_CHAR in ec_optarch:
                 error_msg = "When setting optarch in the easyconfig (found %s), " % ec_optarch
@@ -346,20 +346,21 @@ def _set_optimal_architecture(self, default_optarch=None):
             optarch = build_option('optarch')
 
         # --optarch is specified with flags to use
-        if optarch is not None and isinstance(optarch, dict):
+        if isinstance(optarch, dict):
             # optarch has been validated as complex string with multiple compilers and converted to a dictionary
             # first try module names, then the family in optarch
-            current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) +
-                                      [getattr(self, 'COMPILER_FAMILY', None)])
+            current_compiler_names = self.COMPILER_MODULE_NAME or []
+            if self.COMPILER_FAMILY:
+                current_compiler_names.append(self.COMPILER_FAMILY)
+            compiler_optarch = None
             for current_compiler in current_compiler_names:
                 if current_compiler in optarch:
-                    optarch = optarch[current_compiler]
+                    compiler_optarch = optarch[current_compiler]
                     break
-            # still a dict: no option for this compiler
-            if isinstance(optarch, dict):
-                optarch = None
+            if compiler_optarch is None:
                 self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.",
-                              current_compiler)
+                              current_compiler_names)
+            optarch = self._pick_optarch_entry(compiler_optarch)
 
         use_generic = False
         if optarch is not None:

test/framework/options.py

@@ -5226,10 +5226,14 @@ def test_parse_optarch(self):
         options.options.optarch = 'Intel:something:somethingelse'
         self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)
 
-        error_msg = "The optarch option contains duplicated entries for compiler"
+        error_msg = "The optarch option contains a duplicated entry for compiler"
         options.options.optarch = 'Intel:something;GCC:somethingelse;Intel:anothersomething'
         self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)
 
+        error_msg = "'x86' of 'x86' is not in allowed values"
+        options.options.optarch = 'Intel,x86:something'
+        self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)
+
         # Check the parsing itself
         gcc_generic_flags = "march=x86-64 -mtune=generic"
         test_cases = [
@@ -5239,7 +5243,26 @@ def test_parse_optarch(self):
             ('Intel:xHost', {'Intel': 'xHost'}),
             ('Intel:GENERIC', {'Intel': 'GENERIC'}),
             ('Intel:xHost;GCC:%s' % gcc_generic_flags, {'Intel': 'xHost', 'GCC': gcc_generic_flags}),
-            ('Intel:;GCC:%s' % gcc_generic_flags, {'Intel': '', 'GCC': gcc_generic_flags}),
+            # Allow empty values and spaces
+            ('Intel:; GCC:%s' % gcc_generic_flags, {'Intel': '', 'GCC': gcc_generic_flags}),
+            # Arch specific values. With Compiler only selection at different places in the list
+            (
+                'Intel,x86_64,Intel,sse:i86iSSE; Intel,x86_64,AMD:i86AMD; Intel,POWER:iPwr; Intel:xHost; '
+                'GCC:GENERIC; Clang:cl; Clang,POWER:clPwr',
+                {
+                    'Intel': {
+                        ('x86_64', 'Intel', 'sse'): 'i86iSSE',
+                        ('x86_64', 'AMD'): 'i86AMD',
+                        'POWER': 'iPwr',
+                        None: 'xHost',
+                    },
+                    'GCC': 'GENERIC',
+                    'Clang': {
+                        None: 'cl',
+                        'POWER': 'clPwr',
+                    },
+                },
+            ),
         ]
 
         for optarch_string, optarch_parsed in test_cases:

test/framework/systemtools.py

@@ -923,16 +923,24 @@ def test_pick_opt_arch(self):
         avx = [st.SSE, st.SSE2, st.AVX]
         avx2 = avx + [st.AVX2]
         avx512 = avx2 + [st.AVX512F]
-        self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, avx2), 'opt-x86')
-        self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, []), 'opt-x86')
-        self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx), 'opt-amd')
-        self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx2), 'opt-amd-avx2')
-        self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx512), 'opt-amd-avx2')
-        self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.AMD, avx2), 'opt-aarch64')
-        self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, avx2), 'opt-aarch64-sse')
-        self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, [st.SSE]), 'opt-aarch64-sse')
-        self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), None)
-        self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), None)
+        for include_none in (False, True):
+            if include_none:
+                # Special fallback option when no entry matches
+                options[None] = 'opt-none'
+            self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, avx2), 'opt-x86')
+            self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, []), 'opt-x86')
+            self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx), 'opt-amd')
+            self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx2), 'opt-amd-avx2')
+            self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx512), 'opt-amd-avx2')
+            self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.AMD, avx2), 'opt-aarch64')
+            self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, avx2), 'opt-aarch64-sse')
+            self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, [st.SSE]), 'opt-aarch64-sse')
+            if include_none:
+                self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), 'opt-none')
+                self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), 'opt-none')
+            else:
+                self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), None)
+                self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), None)
 
     def test_check_os_dependency(self):
         """Test check_os_dependency."""