From 4d9f077e0012718068300704b0dc02367b642190 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 00:42:17 +0200 Subject: [PATCH 1/6] use new ModEnvVarType.STRICT_PATH_WITH_FILES for CMAKE_LIBRARY_PATH to avoid spurious default paths injected into generated module files --- easybuild/framework/easyblock.py | 19 ++++++++++++++++--- easybuild/tools/modules.py | 10 +++++++--- test/framework/easyblock.py | 6 ++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 38fb3b0e7b..bc45b352a9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1770,6 +1770,12 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI - PATH_WITH_FILES: must contain at least one file in them (default) - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory """ + populated_path_types = ( + ModEnvVarType.PATH_WITH_FILES, + ModEnvVarType.PATH_WITH_TOP_FILES, + ModEnvVarType.STRICT_PATH_WITH_FILES, + ) + if os.path.isabs(search_path): abs_glob = search_path else: @@ -1780,10 +1786,17 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI retained_search_paths = [] for abs_path in exp_search_paths: - check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES) - if os.path.isdir(abs_path) and check_dir_files: + # avoid going through symlink for strict path types + if path_type is ModEnvVarType.STRICT_PATH_WITH_FILES and abs_path != os.path.realpath(abs_path): + self.log.debug( + f"Discarded strict search path '{search_path} of type '{path_type}' that does not correspond " + f"to its real path: {abs_path}" + ) + continue + + if os.path.isdir(abs_path) and path_type in populated_path_types: # only retain paths to directories that contain at least one file - recursive = path_type == ModEnvVarType.PATH_WITH_FILES + recursive = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES) if not dir_contains_files(abs_path, recursive=recursive): self.log.debug("Discarded search path to empty directory: %s", abs_path) continue diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 97d1d5ae15..1b1b914660 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -144,8 +144,10 @@ class ModEnvVarType(Enum): one or more files - PATH_WITH_TOP_FILES: (list of) of paths to existing directories containing one or more files in its top directory + - STRICT_PATH_WITH_FILES: (list of) of paths to existing directories + containing one or more files, given paths must correspond to real paths - """ - STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES = range(0, 4) + STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES, STRICT_PATH_WITH_FILES = range(0, 5) class ModuleEnvironmentVariable: @@ -239,6 +241,7 @@ def is_path(self): ModEnvVarType.PATH, ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES, + ModEnvVarType.STRICT_PATH_WITH_FILES, ] return self.type in path_like_types @@ -286,7 +289,8 @@ def __init__(self, aliases=None): self._env_vars = {} self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')] self.CLASSPATH = ['*.jar'] - self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64 + # CMAKE_LIBRARY_PATH only needed for installations outside of 'lib' + self.CMAKE_LIBRARY_PATH = {'contents': ['lib64'], 'var_type': "STRICT_PATH_WITH_FILES"} self.CMAKE_PREFIX_PATH = [''] self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS] self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS @@ -374,7 +378,7 @@ def _set_module_environment_variable(self, name, value): if not self.regex['env_var_name'].match(name): raise EasyBuildError( "Name of ModuleLoadEnvironment attribute does not conform to shell naming rules, " - f"it must only have upper-case letters and underscores: '{name}'" + f"it must only have upper-case letters, numbers and underscores: '{name}'" ) if not isinstance(value, dict): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 0cbefe3387..6a5586d91a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -490,16 +490,20 @@ def test_make_module_req(self): # -- No Files if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib$", guess, re.M)) elif get_module_syntax() == 'Lua': self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) + self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib"))', guess) # -- With files write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test') with eb.module_generator.start_module_creation(): guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib$", guess, re.M)) elif get_module_syntax() == 'Lua': self.assertIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) + self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib"))', guess) # -- With files in lib and lib64 symlinks to lib write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) @@ -508,8 +512,10 @@ def test_make_module_req(self): guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib$", guess, re.M)) elif get_module_syntax() == 'Lua': self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) + self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib"))', guess) # With files in /lib and /lib64 symlinked to /lib there should be exactly 1 entry for (LD_)LIBRARY_PATH # pointing to /lib From 8a41852b0a3606a87f9698d743c0769a5027cdff Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 01:15:25 +0200 Subject: [PATCH 2/6] update test_toy_module_fulltxt to new strcit type for CMAKE_LIBRARY_PATH --- test/framework/toy_build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5f02ea2f1e..fbebfaa869 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1642,7 +1642,6 @@ def test_toy_module_fulltxt(self): r'', r'conflict\("toy"\)', r'', - r'prepend_path\("CMAKE_LIBRARY_PATH", pathJoin\(root, "lib"\)\)', r'prepend_path\("CMAKE_PREFIX_PATH", root\)', r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)', r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)', @@ -1684,7 +1683,6 @@ def test_toy_module_fulltxt(self): r'', r'conflict toy', r'', - r'prepend-path CMAKE_LIBRARY_PATH \$root/lib', r'prepend-path CMAKE_PREFIX_PATH \$root', r'prepend-path LD_LIBRARY_PATH \$root/lib', r'prepend-path LIBRARY_PATH \$root/lib', From a343eac552a2d9e42d2ba05c76dd51ca24b016d2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 14:29:37 +0200 Subject: [PATCH 3/6] migrate EasyBlock.expand_module_search_path into ModuleEnvironmentVariable.expand_paths --- easybuild/framework/easyblock.py | 75 ++---------------------------- easybuild/tools/modules.py | 79 +++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 71 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bc45b352a9..1ef01abe24 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -88,11 +88,10 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files -from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file -from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url -from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs -from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum -from easybuild.tools.filetools import weld_paths, write_file +from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate +from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path +from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir +from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file from easybuild.tools.hooks import ( BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP, @@ -1683,10 +1682,7 @@ def make_module_req(self, fake=False): if self.dry_run: self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}") else: - mod_req_paths = [ - expanded_path for unexpanded_path in search_paths - for expanded_path in self.expand_module_search_path(unexpanded_path, path_type=search_paths.type) - ] + mod_req_paths = search_paths.expand_paths(self.installdir) if mod_req_paths: mod_req_paths = nub(mod_req_paths) # remove duplicates @@ -1758,67 +1754,6 @@ def inject_module_extra_paths(self): msg += f"and paths='{env_var}'" self.log.debug(msg) - def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES): - """ - Expand given path glob and return list of suitable paths to be used as search paths: - - Paths must point to existing files/directories - - Relative paths are relative to installation prefix root and are kept relative after expansion - - Absolute paths are kept as absolute paths after expansion - - Follow symlinks and resolve their paths (avoids duplicate paths through symlinks) - - :path_type: ModEnvVarType that controls requirements for population of directories - - PATH: no requirements, can be empty - - PATH_WITH_FILES: must contain at least one file in them (default) - - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory - """ - populated_path_types = ( - ModEnvVarType.PATH_WITH_FILES, - ModEnvVarType.PATH_WITH_TOP_FILES, - ModEnvVarType.STRICT_PATH_WITH_FILES, - ) - - if os.path.isabs(search_path): - abs_glob = search_path - else: - real_installdir = os.path.realpath(self.installdir) - abs_glob = os.path.join(real_installdir, search_path) - - exp_search_paths = glob.glob(abs_glob, recursive=True) - - retained_search_paths = [] - for abs_path in exp_search_paths: - # avoid going through symlink for strict path types - if path_type is ModEnvVarType.STRICT_PATH_WITH_FILES and abs_path != os.path.realpath(abs_path): - self.log.debug( - f"Discarded strict search path '{search_path} of type '{path_type}' that does not correspond " - f"to its real path: {abs_path}" - ) - continue - - if os.path.isdir(abs_path) and path_type in populated_path_types: - # only retain paths to directories that contain at least one file - recursive = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES) - if not dir_contains_files(abs_path, recursive=recursive): - self.log.debug("Discarded search path to empty directory: %s", abs_path) - continue - - if os.path.isabs(search_path): - retain_path = abs_path - else: - # recover relative path - retain_path = os.path.relpath(os.path.realpath(abs_path), start=real_installdir) - if retain_path == '.': - retain_path = '' # use empty string to represent root of install dir - - if retain_path.startswith('..' + os.path.sep): - raise EasyBuildError( - f"Expansion of search path glob pattern '{search_path}' resulted in a relative path " - f"pointing outside of install directory: {retain_path}" - ) - - retained_search_paths.append(retain_path) - - return retained_search_paths - def make_module_req_guess(self): """ A dictionary of common search path variables to be loaded by environment modules diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 1b1b914660..90599af8ba 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -50,7 +50,8 @@ from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars -from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file +from easybuild.tools.filetools import convert_name, dir_contains_files, mkdir, normalize_path, path_matches, read_file +from easybuild.tools.filetools import which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -237,6 +238,7 @@ def remove(self, *args): @property def is_path(self): + """Return True for any ModEnvVarType that is a path""" path_like_types = [ ModEnvVarType.PATH, ModEnvVarType.PATH_WITH_FILES, @@ -245,6 +247,81 @@ def is_path(self): ] return self.type in path_like_types + def expand_paths(self, parent): + """ + Expand path glob into list of unique corresponding real paths. + General behaviour: + - Only expand path-like variables + - Paths must point to existing files/directories + - Resolve paths following symlinks into real paths to avoid duplicate + paths through symlinks + - Relative paths are expanded on given parent folder and are kept + relative after expansion + - Absolute paths are kept as absolute paths after expansion + Follow requirements based on current type (ModEnvVarType): + - PATH: no requirements, must exist but can be empty + - PATH_WITH_FILES: must contain at least one file anywhere in subtree + - PATH_WITH_TOP_FILES: must contain files in top level directory of path + - STRICT_PATH_WITH_FILES: given path must expand into its real path and + contain files anywhere in subtree + """ + if not self.is_path: + return None + + populated_path_types = ( + ModEnvVarType.PATH_WITH_FILES, + ModEnvVarType.PATH_WITH_TOP_FILES, + ModEnvVarType.STRICT_PATH_WITH_FILES, + ) + + retained_expanded_paths = [] + real_parent = os.path.realpath(parent) + + for path_glob in self.contents: + abs_glob = path_glob + if not os.path.isabs(path_glob): + abs_glob = os.path.join(real_parent, path_glob) + + expanded_paths = glob.glob(abs_glob, recursive=True) + + for exp_path in expanded_paths: + real_path = os.path.realpath(exp_path) + + if self.type is ModEnvVarType.STRICT_PATH_WITH_FILES and exp_path != real_path: + # avoid going through symlink for strict path types + self.log.debug( + f"Discarded search path '{exp_path} of type '{self.type}' as it does not correspond " + f"to its real path: {real_path}" + ) + continue + + if os.path.isdir(exp_path) and self.type in populated_path_types: + # only retain paths to directories that contain at least one file + recursive = self.type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES) + if not dir_contains_files(exp_path, recursive=recursive): + self.log.debug(f"Discarded search path '{exp_path}' of type '{self.type}' to empty directory.") + continue + + retain_path = exp_path # no discards, we got a keeper + + if not os.path.isabs(path_glob): + # recover relative path + retain_path = os.path.relpath(real_path, start=real_parent) + # modules use empty string to represent root of install dir + if retain_path == '.': + retain_path = '' + + if retain_path.startswith('..' + os.path.sep): + raise EasyBuildError( + f"Expansion of search path glob pattern '{path_glob}' resulted in a relative path " + f"pointing outside of parent directory: {retain_path}" + ) + + if retain_path not in retained_expanded_paths: + retained_expanded_paths.append(retain_path) + + return retained_expanded_paths + class ModuleLoadEnvironment: """ From 2abcf3c985038185b5e286f36588c28b8e72d66e Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 14:32:09 +0200 Subject: [PATCH 4/6] replace test_expand_module_search_path with test_module_environment_variable_expand_paths --- test/framework/easyblock.py | 130 +--------------------- test/framework/modules.py | 215 +++++++++++++++++++++++++++++++++++- 2 files changed, 215 insertions(+), 130 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6a5586d91a..6eac22a570 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -682,7 +682,7 @@ def test_make_module_req(self): eb.module_load_environment.PATH = ['bin'] with eb.module_generator.start_module_creation(): - err_regex = "Expansion of search path glob.*pointing outside of install directory.*" + err_regex = "Expansion of search path glob.*pointing outside of parent directory.*" self.assertErrorRegex(EasyBuildError, err_regex, eb.make_module_req) # Test modextrapaths: with absolute + empty paths, appending and custom delimiters @@ -3492,134 +3492,6 @@ def test_create_easyblock_without_logfile(self): os.remove(eb.logfile) - def test_expand_module_search_path(self): - """Testcase for expand_module_search_path""" - top_dir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(top_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - eb = EasyBlock(EasyConfig(toy_ec)) - eb.installdir = config.install_path() - test_emsp = eb.expand_module_search_path # shortcut - - # create test directories and files - os.makedirs(eb.installdir) - test_directories = ( - 'empty_dir', - 'dir_empty_subdir', - ('dir_empty_subdir', 'empty_subdir'), - 'dir_with_file', - 'dir_full_subdirs', - ('dir_full_subdirs', 'subdir1'), - ('dir_full_subdirs', 'subdir2'), - ) - for path in test_directories: - path_components = (path, ) if isinstance(path, str) else path - os.mkdir(os.path.join(eb.installdir, *path_components)) - - write_file(os.path.join(eb.installdir, 'dir_with_file', 'file.txt'), 'test file') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') - - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH), ["empty_dir"]) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH), ["dir_empty_subdir"]) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_FILES), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_TOP_FILES), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH), ["dir_full_subdirs"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_FILES), ["dir_full_subdirs"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_TOP_FILES), []) - - # test globs - ref_expanded_paths = ["dir_empty_subdir/empty_subdir"] - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH), ref_expanded_paths) - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) - ref_expanded_paths = ["dir_full_subdirs/subdir1", "dir_full_subdirs/subdir2"] - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH)), ref_expanded_paths) - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_FILES)), ref_expanded_paths) - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_TOP_FILES)), ref_expanded_paths) - ref_expanded_paths = ["dir_full_subdirs/subdir2/file21.txt"] - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH), ref_expanded_paths) - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_FILES), ref_expanded_paths) - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_TOP_FILES), ref_expanded_paths) - self.assertEqual(test_emsp("nonexistent/*", True), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) - - # test just one lib directory - os.mkdir(os.path.join(eb.installdir, "lib")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) - write_file(os.path.join(eb.installdir, "lib", "libtest.so"), "not actually a lib") - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - - # test both lib and lib64 directories - os.mkdir(os.path.join(eb.installdir, "lib64")) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"]) - - # test lib64 symlinked to lib - remove_dir(os.path.join(eb.installdir, "lib64")) - os.symlink("lib", os.path.join(eb.installdir, "lib64")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib", "lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib", "lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib", "lib"]) - - # test lib symlinked to lib64 - remove_dir(os.path.join(eb.installdir, "lib")) - remove_file(os.path.join(eb.installdir, "lib64")) - os.mkdir(os.path.join(eb.installdir, "lib64")) - write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") - os.symlink("lib64", os.path.join(eb.installdir, "lib")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib64"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64", "lib64"]) - - # test both lib and lib64 symlinked to some other folder - remove_dir(os.path.join(eb.installdir, "lib64")) - remove_file(os.path.join(eb.installdir, "lib")) - os.mkdir(os.path.join(eb.installdir, "some_dir")) - write_file(os.path.join(eb.installdir, "some_dir", "libtest.so"), "not actually a lib") - os.symlink("some_dir", os.path.join(eb.installdir, "lib")) - os.symlink("some_dir", os.path.join(eb.installdir, "lib64")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["some_dir"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["some_dir", "some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["some_dir", "some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["some_dir", "some_dir"]) - def suite(): """ return all the tests in this file """ diff --git a/test/framework/modules.py b/test/framework/modules.py index 6080366fa6..59345a6acc 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -43,7 +43,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file @@ -1682,6 +1682,219 @@ def test_module_environment_variable(self): self.assertEqual(mod_envar.contents, ['new_path_1', 'new_path_2']) self.assertRaises(TypeError, mod_envar.update, 'arg1', 'arg2') + def test_module_environment_variable_expand_paths(self): + """Testcase for ModuleEnvironmentVariable.expand_paths""" + + # create test directories and files + installdir = tempfile.mkdtemp() + test_directories = ( + 'empty_dir', + 'dir_empty_subdir', + ('dir_empty_subdir', 'empty_subdir'), + 'dir_with_file', + 'dir_full_subdirs', + ('dir_full_subdirs', 'subdir1'), + ('dir_full_subdirs', 'subdir2'), + ) + for path in test_directories: + path_components = (path, ) if isinstance(path, str) else path + os.mkdir(os.path.join(installdir, *path_components)) + + write_file(os.path.join(installdir, 'dir_with_file', 'file.txt'), 'test file') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') + + def assert_expanded_paths(test_paths, references, installdir): + mod_envar = mod.ModuleEnvironmentVariable(test_paths) + for var_type, ref_value in references: + mod_envar.type = var_type + self.assertEqual(sorted(mod_envar.expand_paths(installdir)), sorted(ref_value)) + + # test simple paths + test_paths = ['nonexistent', 'empty_dir', 'dir_empty_subdir', 'dir_with_file', 'dir_full_subdirs'] + references = ( + (mod.ModEnvVarType.PATH, ['empty_dir', 'dir_empty_subdir', 'dir_with_file', 'dir_full_subdirs']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['dir_with_file', 'dir_full_subdirs']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['dir_with_file']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['dir_with_file', 'dir_full_subdirs']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test globs + test_paths = ['dir_empty_subdir/*', 'dir_full_subdirs/*', 'dir_full_subdirs/subdir2/*', 'nonexistent/*'] + references = ( + (mod.ModEnvVarType.PATH, [ + "dir_empty_subdir/empty_subdir", + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.PATH_WITH_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test just one lib directory + os.mkdir(os.path.join(installdir, "lib")) + + test_paths = ['lib', 'lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, []), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, []), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + + write_file(os.path.join(installdir, 'lib', 'libtest.so'), "not actually a lib") + test_paths = ['lib', 'lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test both lib and lib64 directories + os.mkdir(os.path.join(installdir, "lib64")) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + write_file(os.path.join(installdir, "lib64", "libtest.so"), "not actually a lib") + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib', 'lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib', 'lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test lib64 symlinked to lib + remove_dir(os.path.join(installdir, "lib64")) + os.symlink("lib", os.path.join(installdir, "lib64")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test lib symlinked to lib64 + remove_dir(os.path.join(installdir, "lib")) + remove_file(os.path.join(installdir, "lib64")) + os.mkdir(os.path.join(installdir, "lib64")) + write_file(os.path.join(installdir, "lib64", "libtest.so"), "not actually a lib") + os.symlink("lib64", os.path.join(installdir, "lib")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test both lib and lib64 symlinked to some other folder + remove_dir(os.path.join(installdir, "lib64")) + remove_file(os.path.join(installdir, "lib")) + os.mkdir(os.path.join(installdir, "some_dir")) + write_file(os.path.join(installdir, "some_dir", "libtest.so"), "not actually a lib") + os.symlink("some_dir", os.path.join(installdir, "lib")) + os.symlink("some_dir", os.path.join(installdir, "lib64")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test folder symlinked to folder outside installdir + os.symlink("/bin", os.path.join(installdir, "external_bin")) + test_paths = ['external_bin'] + mod_envar = mod.ModuleEnvironmentVariable(test_paths) + mod_envar.type = mod.ModEnvVarType.PATH + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.PATH_WITH_FILES + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.PATH_WITH_TOP_FILES + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.STRICT_PATH_WITH_FILES + self.assertEqual(mod_envar.expand_paths(installdir), []) + def test_module_load_environment(self): """Test for ModuleLoadEnvironment object""" mod_load_env = mod.ModuleLoadEnvironment() From 144c28a94ca784048cc29fa510627030123e2607 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 14:41:50 +0200 Subject: [PATCH 5/6] remove unused module from test.framework.modules --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 59345a6acc..dc111930ac 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -43,7 +43,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file From df75916b38b152717794ca719b6f9fd25c4f065a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 25 Apr 2025 14:44:08 +0200 Subject: [PATCH 6/6] remove unused module from test.framework.easyblock --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6eac22a570..4b88724502 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -54,7 +54,7 @@ from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file from easybuild.tools.filetools import symlink, verify_checksum, write_file from easybuild.tools.module_generator import module_generator -from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches +from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild