From 9175e980e4e91cb13967910027340126b804556f Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 29 Oct 2024 16:16:28 +0100 Subject: [PATCH 1/3] Added option --- easybuild/main.py | 12 +++++++++++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index f91ebbbbec..9d24e6c4a5 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -61,7 +61,7 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak -from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option +from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option, update_build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software from easybuild.tools.environment import restore_env @@ -691,6 +691,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr index = load_index(options.create_index) print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) + if options.developer: + pth = build_option('developer') + if not os.path.exists(pth): + raise EasyBuildError("Developer mode path %s does not exist" % pth) + + pth = os.path.abspath(pth) + options.developer = pth + update_build_option('developer', pth) + print_msg("Developer mode running from %s" % build_option('developer'), log=_log) + # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.add_pr_labels, diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 664080d8f2..e788a382ad 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -231,6 +231,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'cuda_cache_dir', 'cuda_cache_maxsize', 'cuda_compute_capabilities', + 'developer', 'dump_test_report', 'easyblock', 'envvars_user_modules', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4068aa1ad6..59de2608e4 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -273,6 +273,7 @@ def basic_options(self): descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ + 'developer': ("This is a test option", None, 'store_or_None', '.'), 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'extended-dry-run': ("Print build environment and (expected) build procedure that will be performed", From d83b813f7dc1a6487849cf2e74cacc396e40cf42 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 30 Oct 2024 16:01:44 +0100 Subject: [PATCH 2/3] Trying with simple copy --- easybuild/framework/easyblock.py | 52 +++++++++++++++++++++++++++----- easybuild/main.py | 12 ++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4d9b4fbdd7..43c10104d9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -191,6 +191,19 @@ def __init__(self, ec, logfile=None): self.cfg = ec else: raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec) + + # are we running in developer mode? + self.developer = build_option('developer') + + # This is needed in case custom easyblocks do a reparse of the EC file (e.g. quantum espresso) + if self.developer and not self.version.endswith('-dev'): + old_vers = ec['version'] + new_vers = old_vers + '-dev' + ec['version'] = new_vers + for key in ['short_mod_name', 'full_mod_name']: + prev = getattr(ec, key) + new = prev.replace(f'/{old_vers}', f'/{new_vers}') + setattr(ec, key, new) # modules interface with default MODULEPATH self.modules_tool = self.cfg.modules_tool @@ -2792,15 +2805,29 @@ def extract_step(self): """ Unpack the source files. """ - for src in self.src: - self.log.info("Unpacking source %s" % src['name']) - srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'], - extra_options=self.cfg['unpack_options'], change_into_dir=False) - change_dir(srcdir) - if srcdir: - self.src[self.src.index(src)]['finalpath'] = srcdir + developer_pth = self.developer + if developer_pth: + if self.start_dir: + dst_name = self.start_dir else: - raise EasyBuildError("Unpacking source %s failed", src['name']) + dst_name = os.path.basename(developer_pth) + dest = os.path.join(self.builddir, dst_name) + copy_dir( + developer_pth, dest, + ignore=lambda pth,elem: [_ for _ in elem if _ in ['.git', '.svn', '.hg']] + ) + self.cfg['start_dir'] = dest + self.log.info("Content of develop path %s copied to %s (new start_dir)", developer_pth, dest) + else: + for src in self.src: + self.log.info("Unpacking source %s" % src['name']) + srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'], + extra_options=self.cfg['unpack_options'], change_into_dir=False) + change_dir(srcdir) + if srcdir: + self.src[self.src.index(src)]['finalpath'] = srcdir + else: + raise EasyBuildError("Unpacking source %s failed", src['name']) def patch_step(self, beginpath=None, patches=None): """ @@ -4192,6 +4219,12 @@ def skip_step(self, step, skippable): skip_test_step = build_option('skip_test_step') skipsteps = self.cfg['skipsteps'] + developer_skip = ( + FETCH_STEP, + PATCH_STEP, + EXTENSIONS_STEP, + ) + # under --skip, sanity check is not skipped cli_skip = self.skip and step != SANITYCHECK_STEP @@ -4227,6 +4260,9 @@ def skip_step(self, step, skippable): elif skip_test_step and step == TEST_STEP: self.log.info("Skipping %s step as requested via skip-test-step", step) skip = True + elif self.developer and step in developer_skip: + self.log.info("Skipping %s step because of --developer", step) + skip = True else: msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " diff --git a/easybuild/main.py b/easybuild/main.py index 9d24e6c4a5..37788f5ed8 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -445,6 +445,18 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only )) + if build_option('developer'): + for ec in easyconfigs: + _ec = ec['ec'] + old_vers = _ec['version'] + new_vers = old_vers + '-dev' + _ec['version'] = new_vers + for key in ['short_mod_name', 'full_mod_name']: + prev = getattr(_ec, key) + new = prev.replace(f'/{old_vers}', f'/{new_vers}') + ec[key] = new + setattr(_ec, key, new) + # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: retained_ecs = skip_available(easyconfigs, modtool) From e9df5c59c27c6b4a4a3107957624e6f1a716a3eb Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 30 Oct 2024 16:06:31 +0100 Subject: [PATCH 3/3] lint --- easybuild/framework/easyblock.py | 10 ++++++---- easybuild/main.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 43c10104d9..813cc322b2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -191,7 +191,7 @@ def __init__(self, ec, logfile=None): self.cfg = ec else: raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec) - + # are we running in developer mode? self.developer = build_option('developer') @@ -2814,15 +2814,17 @@ def extract_step(self): dest = os.path.join(self.builddir, dst_name) copy_dir( developer_pth, dest, - ignore=lambda pth,elem: [_ for _ in elem if _ in ['.git', '.svn', '.hg']] + ignore=lambda pth, elem: [_ for _ in elem if _ in ['.git', '.svn', '.hg']] ) self.cfg['start_dir'] = dest self.log.info("Content of develop path %s copied to %s (new start_dir)", developer_pth, dest) else: for src in self.src: self.log.info("Unpacking source %s" % src['name']) - srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'], - extra_options=self.cfg['unpack_options'], change_into_dir=False) + srcdir = extract_file( + src['path'], self.builddir, cmd=src['cmd'], + extra_options=self.cfg['unpack_options'], change_into_dir=False + ) change_dir(srcdir) if srcdir: self.src[self.src.index(src)]['finalpath'] = srcdir diff --git a/easybuild/main.py b/easybuild/main.py index 37788f5ed8..aff4fc6e82 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -707,7 +707,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr pth = build_option('developer') if not os.path.exists(pth): raise EasyBuildError("Developer mode path %s does not exist" % pth) - + pth = os.path.abspath(pth) options.developer = pth update_build_option('developer', pth)