move EasyBlock.expand_module_search_path into ModuleEnvironmentVariable.expand_paths

easybuild/framework/easyblock.py

@@ -88,11 +88,10 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock
 from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files
-from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs
-from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum
-from easybuild.tools.filetools import weld_paths, write_file
+from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate
+from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path
+from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
+from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
     BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
     MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
@@ -1683,10 +1682,7 @@ def make_module_req(self, fake=False):
                 if self.dry_run:
                     self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}")
             else:
-                mod_req_paths = [
-                    expanded_path for unexpanded_path in search_paths
-                    for expanded_path in self.expand_module_search_path(unexpanded_path, path_type=search_paths.type)
-                ]
+                mod_req_paths = search_paths.expand_paths(self.installdir)
 
             if mod_req_paths:
                 mod_req_paths = nub(mod_req_paths)  # remove duplicates
@@ -1758,54 +1754,6 @@ def inject_module_extra_paths(self):
                 msg += f"and paths='{env_var}'"
                 self.log.debug(msg)
 
-    def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES):
-        """
-        Expand given path glob and return list of suitable paths to be used as search paths:
-            - Paths must point to existing files/directories
-            - Relative paths are relative to installation prefix root and are kept relative after expansion
-            - Absolute paths are kept as absolute paths after expansion
-            - Follow symlinks and resolve their paths (avoids duplicate paths through symlinks)
-            - :path_type: ModEnvVarType that controls requirements for population of directories
-              - PATH: no requirements, can be empty
-              - PATH_WITH_FILES: must contain at least one file in them (default)
-              - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory
-        """
-        if os.path.isabs(search_path):
-            abs_glob = search_path
-        else:
-            real_installdir = os.path.realpath(self.installdir)
-            abs_glob = os.path.join(real_installdir, search_path)
-
-        exp_search_paths = glob.glob(abs_glob, recursive=True)
-
-        retained_search_paths = []
-        for abs_path in exp_search_paths:
-            check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES)
-            if os.path.isdir(abs_path) and check_dir_files:
-                # only retain paths to directories that contain at least one file
-                recursive = path_type == ModEnvVarType.PATH_WITH_FILES
-                if not dir_contains_files(abs_path, recursive=recursive):
-                    self.log.debug("Discarded search path to empty directory: %s", abs_path)
-                    continue
-
-            if os.path.isabs(search_path):
-                retain_path = abs_path
-            else:
-                # recover relative path
-                retain_path = os.path.relpath(os.path.realpath(abs_path), start=real_installdir)
-                if retain_path == '.':
-                    retain_path = ''  # use empty string to represent root of install dir
-
-            if retain_path.startswith('..' + os.path.sep):
-                raise EasyBuildError(
-                    f"Expansion of search path glob pattern '{search_path}' resulted in a relative path "
-                    f"pointing outside of install directory: {retain_path}"
-                )
-
-            retained_search_paths.append(retain_path)
-
-        return retained_search_paths
-
     def make_module_req_guess(self):
         """
         A dictionary of common search path variables to be loaded by environment modules

easybuild/tools/modules.py

@@ -50,7 +50,8 @@
 from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET
 from easybuild.tools.config import build_option, get_modules_tool, install_path
 from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars
-from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
+from easybuild.tools.filetools import convert_name, dir_contains_files, mkdir, normalize_path, path_matches, read_file
+from easybuild.tools.filetools import which, write_file
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
@@ -144,8 +145,10 @@ class ModEnvVarType(Enum):
       one or more files
     - PATH_WITH_TOP_FILES: (list of) of paths to existing directories
       containing one or more files in its top directory
+    - STRICT_PATH_WITH_FILES: (list of) of paths to existing directories
+      containing one or more files, given paths must correspond to real paths
     - """
-    STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES = range(0, 4)
+    STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES, STRICT_PATH_WITH_FILES = range(0, 5)
 
 
 class ModuleEnvironmentVariable:
@@ -235,13 +238,90 @@ def remove(self, *args):
 
     @property
     def is_path(self):
+        """Return True for any ModEnvVarType that is a path"""
         path_like_types = [
             ModEnvVarType.PATH,
             ModEnvVarType.PATH_WITH_FILES,
             ModEnvVarType.PATH_WITH_TOP_FILES,
+            ModEnvVarType.STRICT_PATH_WITH_FILES,
         ]
         return self.type in path_like_types
 
+    def expand_paths(self, parent):
+        """
+        Expand path glob into list of unique corresponding real paths.
+        General behaviour:
+        - Only expand path-like variables
+        - Paths must point to existing files/directories
+        - Resolve paths following symlinks into real paths to avoid duplicate
+          paths through symlinks
+        - Relative paths are expanded on given parent folder and are kept
+          relative after expansion
+        - Absolute paths are kept as absolute paths after expansion
+        Follow requirements based on current type (ModEnvVarType):
+          - PATH: no requirements, must exist but can be empty
+          - PATH_WITH_FILES: must contain at least one file anywhere in subtree
+          - PATH_WITH_TOP_FILES: must contain files in top level directory of path
+          - STRICT_PATH_WITH_FILES: given path must expand into its real path and
+            contain files anywhere in subtree
+        """
+        if not self.is_path:
+            return None
+
+        populated_path_types = (
+            ModEnvVarType.PATH_WITH_FILES,
+            ModEnvVarType.PATH_WITH_TOP_FILES,
+            ModEnvVarType.STRICT_PATH_WITH_FILES,
+        )
+
+        retained_expanded_paths = []
+        real_parent = os.path.realpath(parent)
+
+        for path_glob in self.contents:
+            abs_glob = path_glob
+            if not os.path.isabs(path_glob):
+                abs_glob = os.path.join(real_parent, path_glob)
+
+            expanded_paths = glob.glob(abs_glob, recursive=True)
+
+            for exp_path in expanded_paths:
+                real_path = os.path.realpath(exp_path)
+
+                if self.type is ModEnvVarType.STRICT_PATH_WITH_FILES and exp_path != real_path:
+                    # avoid going through symlink for strict path types
+                    self.log.debug(
+                        f"Discarded search path '{exp_path} of type '{self.type}' as it does not correspond "
+                        f"to its real path: {real_path}"
+                    )
+                    continue
+
+                if os.path.isdir(exp_path) and self.type in populated_path_types:
+                    # only retain paths to directories that contain at least one file
+                    recursive = self.type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES)
+                    if not dir_contains_files(exp_path, recursive=recursive):
+                        self.log.debug(f"Discarded search path '{exp_path}' of type '{self.type}' to empty directory.")
+                        continue
+
+                retain_path = exp_path  # no discards, we got a keeper
+
+                if not os.path.isabs(path_glob):
+                    # recover relative path
+                    retain_path = os.path.relpath(real_path, start=real_parent)
+                    # modules use empty string to represent root of install dir
+                    if retain_path == '.':
+                        retain_path = ''
+
+                if retain_path.startswith('..' + os.path.sep):
+                    raise EasyBuildError(
+                        f"Expansion of search path glob pattern '{path_glob}' resulted in a relative path "
+                        f"pointing outside of parent directory: {retain_path}"
+                    )
+
+                if retain_path not in retained_expanded_paths:
+                    retained_expanded_paths.append(retain_path)
+
+        return retained_expanded_paths
+
 
 class ModuleLoadEnvironment:
     """
@@ -286,7 +366,8 @@ def __init__(self, aliases=None):
         self._env_vars = {}
         self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')]
         self.CLASSPATH = ['*.jar']
-        self.CMAKE_LIBRARY_PATH = ['lib64']  # only needed for installations with standalone lib64
+        # CMAKE_LIBRARY_PATH only needed for installations outside of 'lib'
+        self.CMAKE_LIBRARY_PATH = {'contents': ['lib64'], 'var_type': "STRICT_PATH_WITH_FILES"}
         self.CMAKE_PREFIX_PATH = ['']
         self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS]
         self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
@@ -374,7 +455,7 @@ def _set_module_environment_variable(self, name, value):
         if not self.regex['env_var_name'].match(name):
             raise EasyBuildError(
                 "Name of ModuleLoadEnvironment attribute does not conform to shell naming rules, "
-                f"it must only have upper-case letters and underscores: '{name}'"
+                f"it must only have upper-case letters, numbers and underscores: '{name}'"
             )
 
         if not isinstance(value, dict):