From c66f721377e37d7c5b705f538460eef2863b64ae Mon Sep 17 00:00:00 2001 From: Alex Shovlin Date: Thu, 3 Apr 2025 10:51:40 -0400 Subject: [PATCH 1/3] Fix installing setup dependencies on newer versions of pip --- .../enhancement-bundledinstaller-63856.json | 5 +++ scripts/install | 31 +++++++------------ scripts/make-bundle | 18 +++++++++-- 3 files changed, 32 insertions(+), 22 deletions(-) create mode 100644 .changes/next-release/enhancement-bundledinstaller-63856.json diff --git a/.changes/next-release/enhancement-bundledinstaller-63856.json b/.changes/next-release/enhancement-bundledinstaller-63856.json new file mode 100644 index 000000000000..788db42894c1 --- /dev/null +++ b/.changes/next-release/enhancement-bundledinstaller-63856.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "bundled-installer", + "description": "Fix installing setup dependencies on newer versions of pip" +} diff --git a/scripts/install b/scripts/install index cac852f01f64..bc7e25690215 100755 --- a/scripts/install +++ b/scripts/install @@ -150,7 +150,7 @@ def pip_install_packages(install_dir): with cd(PACKAGES_DIR): run( - '{} -m pip install {} --find-links file://{} {}'.format( + '{} -m pip install {} --find-links {} {}'.format( python, INSTALL_ARGS, PACKAGES_DIR, cli_tarball ) ) @@ -160,24 +160,17 @@ def _install_setup_deps(python, setup_package_dir): # Some packages declare `setup_requires`, which is a list of dependencies # to be used at setup time. These need to be installed before anything # else, and pip doesn't manage them. We have to manage this ourselves - # so for now we're explicitly installing the one setup_requires package - # we need. This comes from python-dateutils. - setuptools_scm_tarball = _get_package_tarball( - setup_package_dir, 'setuptools_scm' - ) - run( - ( - '{} -m pip install --no-binary :all: --no-cache-dir --no-index ' - '--find-links file://{} {}' - ).format(python, setup_package_dir, setuptools_scm_tarball) - ) - wheel_tarball = _get_package_tarball(setup_package_dir, 'wheel') - run( - ( - '{} -m pip install --no-binary :all: --no-cache-dir --no-index ' - '--find-links file://{} {}' - ).format(python, setup_package_dir, wheel_tarball) - ) + # so for now we're explicitly installing setuptools_scm which is needed for + # python-dateutils. We're also now installing setuptools since its no + # longer installed alongside pip for 3.12+. + for package in ['setuptools-', 'wheel', 'setuptools_scm']: + # these are actually wheels, but the bundle lookup logic is the same + tarball = _get_package_tarball(setup_package_dir, package) + run( + '{} -m pip install {} --find-links {} {}'.format( + python, INSTALL_ARGS, PACKAGES_DIR, tarball + ) + ) def create_symlink(real_location, symlink_name): diff --git a/scripts/make-bundle b/scripts/make-bundle index 6ca3dbbcf595..c9a5863dd50b 100755 --- a/scripts/make-bundle +++ b/scripts/make-bundle @@ -33,8 +33,9 @@ PINNED_RUNTIME_DEPS = [ ('colorama', '0.4.5'), ] BUILDTIME_DEPS = [ + ('setuptools', '75.3.2'), # 75.4 dropped 3.8 support ('setuptools-scm', '3.3.3'), - ('wheel', '0.33.6'), + ('wheel', '0.45.1'), # 46.0 dropped 3.8 support ] PIP_DOWNLOAD_ARGS = '--no-build-isolation --no-binary :all:' @@ -84,6 +85,16 @@ def download_package_tarballs(dirname, packages): )) +def download_package_wheels(dirname, packages): + with cd(dirname): + for package, package_version in packages: + run( + '%s -m pip download %s==%s --no-build-isolation ' + '--only-binary :all:' + % (sys.executable, package, package_version) + ) + + def download_cli_deps(scratch_dir, packages): # pip download will always download a more recent version of a package # even if one exists locally. The list of packages supplied in `packages` @@ -174,9 +185,10 @@ def main(): # manually install them. We isolate them to a particular directory so we # can run the install before the things they're dependent on. We have to do # this because pip won't actually find them since it doesn't handle build - # dependencies. + # dependencies. We use wheels for this, to avoid bootstrapping setuptools + # in 3.12+ where its no longer included by default. setup_dir = os.path.join(package_dir, 'setup') - download_package_tarballs( + download_package_wheels( setup_dir, packages=BUILDTIME_DEPS, ) From 4668b956f890b1047ef1f97da3eb4458a8bb31f0 Mon Sep 17 00:00:00 2001 From: Alex Shovlin Date: Wed, 9 Apr 2025 10:34:57 -0400 Subject: [PATCH 2/3] Add GitHub Action to test bundle installation --- .github/workflows/run-bundle-test.yml | 27 ++++++++++++ scripts/ci/test-bundle | 63 +++++++++++++++++++++++++++ scripts/make-bundle | 8 ++-- 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/run-bundle-test.yml create mode 100755 scripts/ci/test-bundle diff --git a/.github/workflows/run-bundle-test.yml b/.github/workflows/run-bundle-test.yml new file mode 100644 index 000000000000..1bae8adc63d6 --- /dev/null +++ b/.github/workflows/run-bundle-test.yml @@ -0,0 +1,27 @@ +name: Run bundle test + +on: + push: + pull_request: + branches-ignore: [ master ] + +jobs: + test-bundle: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, macOS-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: python scripts/ci/install + - name: Install additional dependencies + run: pip install virtualenv==16.3.0 setuptools-scm==3.3.3 # same as internal generate-bundle.ts + - name: Test the bundle + run: python scripts/ci/test-bundle diff --git a/scripts/ci/test-bundle b/scripts/ci/test-bundle new file mode 100755 index 000000000000..81bf4c4e4c56 --- /dev/null +++ b/scripts/ci/test-bundle @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# Don't run tests from the root repo dir. +# We want to ensure we're importing from the installed +# binary package not from the CWD. + +import os +import re +from contextlib import contextmanager +from subprocess import check_output + +_dname = os.path.dirname + +REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) + + +@contextmanager +def cd(path): + """Change directory while inside context manager.""" + cwd = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(cwd) + + +def run(command): + print(f'Running {command}') + return check_output(command, shell=True) + + +def run_make_bundle(): + """ + Builds the bundled installer, and returns its path + """ + output = run(f'{REPO_ROOT}/scripts/make-bundle') + match = re.search( + r'Zipped bundle installer is at: (.+?\.zip)', output.decode('utf-8') + ) + if not match: + raise RuntimeError("Could not find bundle path in make-bundle output") + + return match.group(1) + + +def install_from_bundle(zip_path): + run(f'unzip -o {bundle_path}') + path_without_zip = bundle_path[:-4] + run( + f'sudo {path_without_zip}/install -i /usr/local/aws -b /usr/local/bin/aws' + ) + + +def verify_installation(): + version_output = run("aws --version") + print(f"Installed AWS CLI version: {version_output}") + + +if __name__ == "__main__": + with cd(os.path.join(REPO_ROOT)): + bundle_path = run_make_bundle() + install_from_bundle(bundle_path) + verify_installation() diff --git a/scripts/make-bundle b/scripts/make-bundle index c9a5863dd50b..6785f418116e 100755 --- a/scripts/make-bundle +++ b/scripts/make-bundle @@ -31,11 +31,13 @@ PINNED_RUNTIME_DEPS = [ # require extra build time dependencies. We are pinning it to # a version that does not need those. ('colorama', '0.4.5'), + # 2.0.0 of urllib3 started requiring hatchling as well + ('urllib3', '1.26.20'), ] BUILDTIME_DEPS = [ - ('setuptools', '75.3.2'), # 75.4 dropped 3.8 support + ('setuptools', '75.4.0'), # start of >= 3.9 ('setuptools-scm', '3.3.3'), - ('wheel', '0.45.1'), # 46.0 dropped 3.8 support + ('wheel', '0.45.1'), # 0.46.0+ requires packaging ] PIP_DOWNLOAD_ARGS = '--no-build-isolation --no-binary :all:' @@ -186,7 +188,7 @@ def main(): # can run the install before the things they're dependent on. We have to do # this because pip won't actually find them since it doesn't handle build # dependencies. We use wheels for this, to avoid bootstrapping setuptools - # in 3.12+ where its no longer included by default. + # in 3.12+ where it's no longer included by default. setup_dir = os.path.join(package_dir, 'setup') download_package_wheels( setup_dir, From 0408ebe220a819aa76f6eebb4b608cf434b8fc92 Mon Sep 17 00:00:00 2001 From: Alex Shovlin Date: Wed, 23 Apr 2025 14:58:23 -0400 Subject: [PATCH 3/3] Validate that package wheels are universal and cleanup --- scripts/ci/test-bundle | 13 +-------- scripts/make-bundle | 63 +++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/scripts/ci/test-bundle b/scripts/ci/test-bundle index 81bf4c4e4c56..52f446f3ce59 100755 --- a/scripts/ci/test-bundle +++ b/scripts/ci/test-bundle @@ -5,25 +5,14 @@ import os import re -from contextlib import contextmanager from subprocess import check_output +from awscli.testutils import cd _dname = os.path.dirname REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) -@contextmanager -def cd(path): - """Change directory while inside context manager.""" - cwd = os.getcwd() - try: - os.chdir(path) - yield - finally: - os.chdir(cwd) - - def run(command): print(f'Running {command}') return check_output(command, shell=True) diff --git a/scripts/make-bundle b/scripts/make-bundle index 6785f418116e..1891c550afd9 100755 --- a/scripts/make-bundle +++ b/scripts/make-bundle @@ -12,6 +12,7 @@ interface for those not familiar with the python ecosystem. """ + import os import sys import subprocess @@ -57,14 +58,14 @@ def cd(dirname): def run(cmd): - sys.stdout.write("Running cmd: %s\n" % cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + sys.stdout.write(f"Running cmd: {cmd}\n") + p = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, stderr = p.communicate() rc = p.wait() if p.returncode != 0: - raise BadRCError("Bad rc (%s) for cmd '%s': %s" % ( - rc, cmd, stderr + stdout)) + raise BadRCError(f"Bad rc ({rc}) for cmd '{cmd}': {stderr + stdout}") return stdout @@ -82,27 +83,33 @@ def create_scratch_dir(): def download_package_tarballs(dirname, packages): with cd(dirname): for package, package_version in packages: - run('%s -m pip download %s==%s %s' % ( - sys.executable, package, package_version, PIP_DOWNLOAD_ARGS - )) + run( + f'{sys.executable} -m pip download {package}=={package_version}' + f' {PIP_DOWNLOAD_ARGS}' + ) def download_package_wheels(dirname, packages): with cd(dirname): for package, package_version in packages: run( - '%s -m pip download %s==%s --no-build-isolation ' - '--only-binary :all:' - % (sys.executable, package, package_version) + f'{sys.executable} -m pip download {package}=={package_version}' + f' --only-binary :all:' ) +def validate_that_wheels_are_universal(dirname): + with cd(dirname): + for wheel_path in os.listdir(): + if not wheel_path.endswith('py3-none-any.whl'): + raise ValueError(f'Found a non universal wheel: {wheel_path}') + + def download_cli_deps(scratch_dir, packages): # pip download will always download a more recent version of a package # even if one exists locally. The list of packages supplied in `packages` # forces the use of a specific runtime dependency. - awscli_dir = os.path.dirname( - os.path.dirname(os.path.abspath(__file__))) + awscli_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) pinned_packages = " ".join( f"{name}=={version}" for (name, version) in packages ) @@ -117,20 +124,21 @@ def _remove_cli_zip(scratch_dir): def add_cli_sdist(scratch_dir): - awscli_dir = os.path.dirname( - os.path.dirname(os.path.abspath(__file__))) + awscli_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if os.path.exists(os.path.join(awscli_dir, 'dist')): shutil.rmtree(os.path.join(awscli_dir, 'dist')) with cd(awscli_dir): - run('%s setup.py sdist' % sys.executable) + run(f'{sys.executable} setup.py sdist') filename = os.listdir('dist')[0] - shutil.move(os.path.join('dist', filename), - os.path.join(scratch_dir, filename)) + shutil.move( + os.path.join('dist', filename), os.path.join(scratch_dir, filename) + ) def create_bootstrap_script(scratch_dir): install_script = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'install') + os.path.dirname(os.path.abspath(__file__)), 'install' + ) shutil.copy(install_script, os.path.join(scratch_dir, 'install')) @@ -152,11 +160,11 @@ def zip_dir(scratch_dir): def verify_preconditions(): # The pip version looks like: # 'pip 1.4.1 from ....' - pip_version = run( - '%s -m pip --version' % sys.executable).strip().split()[1] + pip_version = run(f'{sys.executable} -m pip --version').strip().split()[1] # Virtualenv version just has the version string: '1.14.5\n' virtualenv_version = run( - '%s -m virtualenv --version' % sys.executable).strip() + f'{sys.executable} -m virtualenv --version' + ).strip() _min_version_required('9.0.1', pip_version, 'pip') _min_version_required('15.1.0', virtualenv_version, 'virtualenv') @@ -169,15 +177,17 @@ def _min_version_required(min_version, actual_version, name): for min_version_part, actual_version_part in zip(min_split, actual_split): if int(actual_version_part) >= int(min_version_part): return - raise ValueError("%s requires at least version %s, but version %s was " - "found." % (name, min_version, actual_version)) + raise ValueError( + f'{name} requires at least version {min_version}, ' + f'but version {actual_version} was found.' + ) def main(): verify_preconditions() scratch_dir = create_scratch_dir() package_dir = os.path.join(scratch_dir, 'packages') - print("Bundle dir at: %s" % scratch_dir) + print(f"Bundle dir at: {scratch_dir}") download_package_tarballs( package_dir, packages=EXTRA_RUNTIME_DEPS, @@ -194,11 +204,12 @@ def main(): setup_dir, packages=BUILDTIME_DEPS, ) + validate_that_wheels_are_universal(setup_dir) download_cli_deps(package_dir, packages=PINNED_RUNTIME_DEPS) add_cli_sdist(package_dir) create_bootstrap_script(scratch_dir) zip_filename = zip_dir(scratch_dir) - print("Zipped bundle installer is at: %s" % zip_filename) + print(f"Zipped bundle installer is at: {zip_filename}") if __name__ == '__main__':