Fix installing setup dependencies for bundled installer on newer versions of pip

.changes/next-release/enhancement-bundledinstaller-63856.json

@@ -0,0 +1,5 @@
+{
+  "type": "bugfix",
+  "category": "bundled-installer",
+  "description": "Fix installing setup dependencies on newer versions of pip"
+}

.github/workflows/run-bundle-test.yml

@@ -0,0 +1,27 @@
+name: Run bundle test
+
+on:
+  push:
+  pull_request:
+    branches-ignore: [ master ]
+
+jobs:
+  test-bundle:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, macOS-latest]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: python scripts/ci/install
+    - name: Install additional dependencies
+      run: pip install virtualenv==16.3.0 setuptools-scm==3.3.3  # same as internal generate-bundle.ts
+    - name: Test the bundle
+      run: python scripts/ci/test-bundle

scripts/ci/test-bundle

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# Don't run tests from the root repo dir.
+# We want to ensure we're importing from the installed
+# binary package not from the CWD.
+
+import os
+import re
+from contextlib import contextmanager
+from subprocess import check_output
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+
+
+@contextmanager
+def cd(path):
+    """Change directory while inside context manager."""
+    cwd = os.getcwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(cwd)
+
+
+def run(command):
+    print(f'Running {command}')
+    return check_output(command, shell=True)
+
+
+def run_make_bundle():
+    """
+    Builds the bundled installer, and returns its path
+    """
+    output = run(f'{REPO_ROOT}/scripts/make-bundle')
+    match = re.search(
+        r'Zipped bundle installer is at: (.+?\.zip)', output.decode('utf-8')
+    )
+    if not match:
+        raise RuntimeError("Could not find bundle path in make-bundle output")
+
+    return match.group(1)
+
+
+def install_from_bundle(zip_path):
+    run(f'unzip -o {bundle_path}')
+    path_without_zip = bundle_path[:-4]
+    run(
+        f'sudo {path_without_zip}/install -i /usr/local/aws -b /usr/local/bin/aws'
+    )
+
+
+def verify_installation():
+    version_output = run("aws --version")
+    print(f"Installed AWS CLI version: {version_output}")
+
+
+if __name__ == "__main__":
+    with cd(os.path.join(REPO_ROOT)):
+        bundle_path = run_make_bundle()
+        install_from_bundle(bundle_path)
+        verify_installation()

scripts/install

@@ -149,7 +149,7 @@ def pip_install_packages(install_dir):
 
     with cd(PACKAGES_DIR):
         run(
-            '{} -m pip install {} --find-links file://{} {}'.format(
+            '{} -m pip install {} --find-links {} {}'.format(
                 python, INSTALL_ARGS, PACKAGES_DIR, cli_tarball
             )
         )
@@ -159,24 +159,17 @@ def _install_setup_deps(python, setup_package_dir):
     # Some packages declare `setup_requires`, which is a list of dependencies
     # to be used at setup time. These need to be installed before anything
     # else, and pip doesn't manage them.  We have to manage this ourselves
-    # so for now we're explicitly installing the one setup_requires package
-    # we need.  This comes from python-dateutils.
-    setuptools_scm_tarball = _get_package_tarball(
-        setup_package_dir, 'setuptools_scm'
-    )
-    run(
-        (
-            '{} -m pip install --no-binary :all: --no-cache-dir --no-index '
-            '--find-links file://{} {}'
-        ).format(python, setup_package_dir, setuptools_scm_tarball)
-    )
-    wheel_tarball = _get_package_tarball(setup_package_dir, 'wheel')
-    run(
-        (
-            '{} -m pip install --no-binary :all: --no-cache-dir --no-index '
-            '--find-links file://{} {}'
-        ).format(python, setup_package_dir, wheel_tarball)
-    )
+    # so for now we're explicitly installing setuptools_scm which is needed for
+    # python-dateutils. We're also now installing setuptools since its no
+    # longer installed alongside pip for 3.12+.
+    for package in ['setuptools-', 'wheel', 'setuptools_scm']:
+        # these are actually wheels, but the bundle lookup logic is the same
+        tarball = _get_package_tarball(setup_package_dir, package)
+        run(
+            '{} -m pip install {} --find-links {} {}'.format(
+                python, INSTALL_ARGS, PACKAGES_DIR, tarball
+            )
+        )
 
 
 def create_symlink(real_location, symlink_name):

scripts/make-bundle

@@ -31,10 +31,13 @@ PINNED_RUNTIME_DEPS = [
     # require extra build time dependencies. We are pinning it to
     # a version that does not need those.
     ('colorama', '0.4.5'),
+    # 2.0.0 of urllib3 started requiring hatchling as well
+    ('urllib3', '1.26.20'),
 ]
 BUILDTIME_DEPS = [
+    ('setuptools', '75.4.0'),  # start of >= 3.9
     ('setuptools-scm', '3.3.3'),
-    ('wheel', '0.33.6'),
+    ('wheel', '0.45.1'),  # 0.46.0+ requires packaging
 ]
 PIP_DOWNLOAD_ARGS = '--no-build-isolation --no-binary :all:'
 
@@ -84,6 +87,16 @@ def download_package_tarballs(dirname, packages):
             ))
 
 
+def download_package_wheels(dirname, packages):
+    with cd(dirname):
+        for package, package_version in packages:
+            run(
+                '%s -m pip download %s==%s --no-build-isolation '
+                '--only-binary :all:'
+                % (sys.executable, package, package_version)
+            )
+
+
 def download_cli_deps(scratch_dir, packages):
     # pip download will always download a more recent version of a package
     # even if one exists locally. The list of packages supplied in `packages`
@@ -174,9 +187,10 @@ def main():
     # manually install them. We isolate them to a particular directory so we
     # can run the install before the things they're dependent on. We have to do
     # this because pip won't actually find them since it doesn't handle build
-    # dependencies.
+    # dependencies. We use wheels for this, to avoid bootstrapping setuptools
+    # in 3.12+ where it's no longer included by default.
     setup_dir = os.path.join(package_dir, 'setup')
-    download_package_tarballs(
+    download_package_wheels(
         setup_dir,
         packages=BUILDTIME_DEPS,
     )