From 69f6bd4884e2fe6dda0934ac4cd17c3a56e7ad12 Mon Sep 17 00:00:00 2001 From: Palash Gandhi <87093175+palash-gandhi@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:36:09 -0700 Subject: [PATCH 1/2] DLPX-94086 LTS 24.04: update linux-pkg for Ubuntu 24.04 appliance PR URL: https://www.github.com/delphix/linux-pkg/pull/349 --- .github/scripts/verify-query-packages.sh | 8 -- README.md | 9 +-- default-package-config.sh | 48 ++++++----- lib/common.sh | 18 ++--- package-lists/build/main.pkgs | 8 +- packages/adoptopenjdk/config.sh | 81 ------------------- packages/bcc/config.sh | 39 --------- packages/bpftrace/config.sh | 42 ---------- packages/connstat/config.sh | 2 +- packages/containerized-masking/config.sh | 6 +- packages/delphix-rust/config.sh | 11 +++ packages/delphix-sso-app/config.sh | 6 +- packages/drgn/config.sh | 2 +- packages/fio/config.sh | 41 ---------- packages/linux-kernel-aws/config.delphix.sh | 6 ++ packages/linux-kernel-azure/config.delphix.sh | 6 ++ packages/linux-kernel-gcp/config.delphix.sh | 6 ++ .../linux-kernel-generic/config.delphix.sh | 6 ++ .../linux-kernel-oracle/config.delphix.sh | 6 ++ packages/make-jpkg/config.sh | 30 ------- packages/masking/config.sh | 6 +- packages/misc-debs/config.sh | 5 +- packages/mold/config.sh | 34 -------- packages/virtualization/config.sh | 6 +- packages/zfs/config.sh | 7 +- resources/delphix_kernel_annotations | 2 - setup.sh | 12 ++- 27 files changed, 102 insertions(+), 351 deletions(-) delete mode 100755 packages/adoptopenjdk/config.sh delete mode 100644 packages/bcc/config.sh delete mode 100644 packages/bpftrace/config.sh delete mode 100644 packages/fio/config.sh delete mode 100644 packages/make-jpkg/config.sh delete mode 100644 packages/mold/config.sh diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index 65cc85f0..c4051171 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -30,14 +30,6 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" test ${#fields[@]} -eq 1 test "${fields[0]}" == 'https://github.com/delphix/zfs.git' -# Expect: "bpftrace bcc true https://github.com/delphix/bpftrace.git" -read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)" -test ${#fields[@]} -eq 4 -test "${fields[0]}" == 'bpftrace' -test "${fields[1]}" == 'bcc' -test "${fields[2]}" == 'true' -test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git' - # Expect that "list all" outputs all directory names under packages/ diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) diff --git a/README.md b/README.md index 4efa6b02..4a5aa0b4 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ projects. ## System Requirements -This framework is intended to be run on an Ubuntu 20.04 system with some basic +This framework is intended to be run on an Ubuntu 24.04 system with some basic developer packages installed, such as git, and passwordless sudo enabled. Note that it will automatically install various build-dependencies on the system, so as a safety precaution it is currently restricted to only run on an AWS instance @@ -280,12 +280,7 @@ of some of the scripts defined above. * **package_S3_URL**: Similar to the package_VAR variables above. This is used to override the default S3 location for where package build-dependencies are - fetched for a given linux-pkg package. For instance, if you are building - bpftrace, which has `PACKAGE_DEPENDENCIES="bcc"` in its config, the - `fetch_dependencies()` stage in the build will fetch the latest build - artifacts of the bcc package from a predetermined S3 location. If you pass - `BCC_S3_URL=s3://path/to/custom/bcc/artifacts` then those artifacts will be - fetched insteasd. + fetched for a given linux-pkg package. * **DELPHIX_PACKAGE_MIRROR_MAIN, DELPHIX_PACKAGE_MIRROR_SECONDARY**: When the [setup.sh](#setupsh) script is run, it will configure the apt sources diff --git a/default-package-config.sh b/default-package-config.sh index 93f962c3..c30d3acd 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -35,11 +35,19 @@ function merge_with_upstream() { # The functions below are specific for the Linux kernel packages # and contain the majority of their common code. # +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version to satisfy the kernel's +# dependency on the rust toolchain. Delphix's rust toolchain is +# supplied via virtual packages and hence must be installed explicitly +# otherwise apt installs the Ubuntu's version of the rust toolchain. +# function kernel_prepare() { logmust install_pkgs \ - equivs \ + "$DEPDIR"/delphix-rust/*.deb \ devscripts \ - kernel-wedge + equivs \ + kernel-wedge \ + gawk } # @@ -53,8 +61,8 @@ function kernel_build() { local platform="$1" # # Note: Extra arguments can overwrite default arguments. - # For example in this function we default skipdbg - # to false, but if we pass "skipdbg=true" as an + # For example in this function we default do_dbgsym_package + # to false, but if we pass "do_dbgsym_package=true" as an # extra argument we will be overwriting this value # to true. This is because when a variable's value # is declared multiple times when invoking the @@ -103,31 +111,33 @@ function kernel_build() { echo "$kernel_version" >"$WORKDIR/artifacts/KERNEL_VERSION" # - # skipdbg=false + # do_dbgsym_package=true # We need debug info for our debugging tools to work. # Don't skip them. + # do_tools_common=false + # We do not need to build linux-tools-common package and we + # install it directly from our package mirror. + # do_tools_host=false + # We do not need to build linux-tools-host package. # uefi_signed=false # This variable defaults to true but since we don't have # any intention and logic to provide signatures for now # we set it to false to avoid any misconfigurations down # the line. - # disable_d_i=true - # This prevents udeb packages from being built as they are - # not consumed by the Delphix Appliance. - # do_dkms_*=false + # do_=false # This disables the build of various out-of-tree kernel modules # that we do not use in our product or that we provide separately. # local debian_rules_args=( - "skipdbg=false" + "do_dbgsym_package=true" + "do_tools_common=false" + "do_tools_host=false" "uefi_signed=false" - "disable_d_i=true" "do_zfs=false" - "do_dkms_nvidia=false" - "do_dkms_nvidia_server=false" - "do_dkms_vbox=false" - "do_dkms_wireguard=false" - "dkms_exclude=v4l2loopback" + "do_ipu6=false" + "do_iwlwifi=false" + "do_v4l2loopback=false" + "do_usbio=false" "flavours=$platform" "abinum=${delphix_abinum}" ) @@ -262,11 +272,7 @@ function kernel_update_upstream() { local tag_prefix_flavour case "${platform}" in generic) - if [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then - tag_prefix_flavour="Ubuntu-hwe" - else - tag_prefix_flavour="Ubuntu" - fi + tag_prefix_flavour="Ubuntu" ;; aws | azure | gcp | oracle) tag_prefix_flavour="Ubuntu-${platform}" diff --git a/lib/common.sh b/lib/common.sh index a75f6cc7..73c1ad53 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -28,7 +28,7 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle" # export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}" -export UBUNTU_DISTRIBUTION="focal" +export UBUNTU_DISTRIBUTION="noble" # # We currently support getting the linux kernel from 3 different sources: @@ -616,6 +616,10 @@ function delphix_revision() { echo "delphix.$(date '+%Y.%m.%d.%H.%M')" } +function compare_versions() { + dpkg --compare-versions "$@" +} + function determine_dependencies_base_url() { [[ -n "$DEPENDENCIES_BASE_URL" ]] && return @@ -1109,21 +1113,13 @@ function get_kernel_version_for_platform_from_apt() { # image for that particular platform. For instance, Ubuntu has a # meta-package for AWS called 'linux-image-aws', which depends on # package 'linux-image-4.15.0-1027-aws'. The latter is the linux image - # for kernel version '4.15.0-1027-aws'. We use this depenency to figure + # for kernel version '4.15.0-1027-aws'. We use this dependency to figure # out the default kernel version for a given platform. # - # The "generic" platform is a special case, since we want to use the - # hwe kernel image instead of the regular generic image. - # # Note that while the default kernel is usually also the latest # available, it is not always the case. # - - if [[ "$platform" != generic ]] && [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then - package="linux-image-${platform}" - else - package="linux-image-${platform}-hwe-20.04" - fi + package="linux-image-${platform}" if [[ "$(apt-cache show --no-all-versions "$package" \ 2>/dev/null | grep Depends)" =~ linux-image-([^,]*-${platform}) ]]; then diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs index d96c1fb9..570c0673 100644 --- a/package-lists/build/main.pkgs +++ b/package-lists/build/main.pkgs @@ -2,9 +2,6 @@ # List of non-kernel packages to be included in the Delphix Appliance. # -adoptopenjdk -bcc -bpftrace challenge-response cloud-init crash-python @@ -13,20 +10,17 @@ delphix-go delphix-platform delphix-rust delphix-sso-app -drgn docker-python-image +drgn dwarves -fio fluentd-gems gdb-python grub2 host-jdks libkdumpfile -make-jpkg makedumpfile masking misc-debs -mold nfs-utils performance-diagnostics ptools diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh deleted file mode 100755 index f9b9ef1f..00000000 --- a/packages/adoptopenjdk/config.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2024 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL=none -PACKAGE_DEPENDENCIES="make-jpkg" - -case $(dpkg-architecture -q DEB_HOST_ARCH 2>/dev/null || echo "none") in -amd64) - _tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u442b06.tar.gz" - _tarfile_sha256="5b0a0145e7790552a9c8767b4680074c4628ec276e5bb278b61d85cf90facafa" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" - ;; -arm64) - _tarfile="OpenJDK8U-jdk_aarch64_linux_hotspot_8u442b06.tar.gz" - _tarfile_sha256="1d1662bd8ca7edc9281c723d9eebafea940e6a41464bdc43a83b564bc974c7e5" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-arm64" - ;; -*) ;; - -esac - -function prepare() { - logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb -} - -function fetch() { - # We exit here rather than above in the architecture detection logic - # to deal with the fact that this file can, during test runs, be - # sourced on platforms where builds are not happening. list-packages - # sources the file to gather information about the package, and this - # is performed on jenkins and macos during test runs. Having the exit - # occur above causes those runs to fail. - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile" - - logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256" -} - -function build() { - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<'$WORKDIR/artifacts/JDK_PATH'" - - echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO" -} diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh deleted file mode 100644 index 17ac5032..00000000 --- a/packages/bcc/config.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" - -UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust cd "$WORKDIR/repo" - # Note: the string to determine the version was copied from bcc's - # debian/rules file. - PACKAGE_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1,p') - - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh deleted file mode 100644 index 79cfa307..00000000 --- a/packages/bpftrace/config.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git" -PACKAGE_DEPENDENCIES="bcc" - -UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" -UPSTREAM_GIT_BRANCH="master" - -function fetch() { - logmust fetch_repo_from_git - logmust git submodule init - logmust git submodule update --recursive -} - -function prepare() { - logmust install_pkgs "$DEPDIR"/bcc/libbcc_*.deb - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 33825120..1c8559a3 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -23,7 +23,7 @@ function prepare() { logmust install_pkgs \ debhelper \ dpkg-dev \ - llvm-12 + llvm-14 logmust install_kernel_headers logmust install_pkgs "$DEPDIR"/dwarves/*.deb } diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 7c480b59..7b2fd926 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -29,17 +29,15 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" SKIP_COPYRIGHTS_CHECK=true function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index c826a12e..4f1f06c0 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -20,5 +20,16 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" function build() { logmust mkdir -p "$WORKDIR/repo" + + # + # Instead of relying on linux-pkg to assign a default version like 1.0.0, set the + # version of the `delphix-rust` package to the version of the `rustc` package. + # This is done to ensure that third-party packages that rely on specific versions + # of `rustc` can correctly compare the version of the `rustc` package supplied by + # `delphix-rust` with their requirement. + # + PACKAGE_VERSION="$(tr -d '\n' <"$WORKDIR/repo/RUSTC_VERSION")" + [[ -n "$PACKAGE_VERSION" ]] || die "Failed to retrieve package version" + logmust dpkg_buildpackage_default } diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 3f880145..79ac9139 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -17,16 +17,14 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { local java_home - java_home=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + java_home="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" logmust sudo ./gradlew "-Dorg.gradle.java.home=$java_home" distDeb logmust sudo mv ./build/distributions/*deb "$WORKDIR/artifacts/" diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 0387a3a7..4a180ca1 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2019, 2020 Delphix +# Copyright 2019, 2025 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fio/config.sh b/packages/fio/config.sh deleted file mode 100644 index 758ff252..00000000 --- a/packages/fio/config.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2022 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fio.git" - -function prepare() { - logmust install_pkgs \ - libaio-dev \ - librdmacm-dev \ - libibverbs-dev \ - librbd-dev \ - libgtk2.0-dev \ - libcairo2-dev \ - libnuma-dev \ - flex \ - bison \ - libglusterfs-dev \ - libpmem-dev \ - libpmemblk-dev -} - -function build() { - logmust cd "$WORKDIR/repo/" - logmust dpkg-buildpackage -b -us -uc - logmust mv "$WORKDIR"/*deb "$WORKDIR/artifacts/" -} diff --git a/packages/linux-kernel-aws/config.delphix.sh b/packages/linux-kernel-aws/config.delphix.sh index 220d14b3..c369203a 100644 --- a/packages/linux-kernel-aws/config.delphix.sh +++ b/packages/linux-kernel-aws/config.delphix.sh @@ -22,6 +22,12 @@ UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/lin # Note: UPSTREAM_GIT_BRANCH is not used here UPSTREAM_GIT_BRANCH="none" +# +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version. +# +PACKAGE_DEPENDENCIES="delphix-rust" + # # Force push required when syncing with upstream because we perform a rebase. # diff --git a/packages/linux-kernel-azure/config.delphix.sh b/packages/linux-kernel-azure/config.delphix.sh index 85a41381..f2e46969 100644 --- a/packages/linux-kernel-azure/config.delphix.sh +++ b/packages/linux-kernel-azure/config.delphix.sh @@ -22,6 +22,12 @@ UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/lin # Note: UPSTREAM_GIT_BRANCH is not used here UPSTREAM_GIT_BRANCH="none" +# +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version. +# +PACKAGE_DEPENDENCIES="delphix-rust" + # # Force push required when syncing with upstream because we perform a rebase. # diff --git a/packages/linux-kernel-gcp/config.delphix.sh b/packages/linux-kernel-gcp/config.delphix.sh index 8503731b..5318c168 100644 --- a/packages/linux-kernel-gcp/config.delphix.sh +++ b/packages/linux-kernel-gcp/config.delphix.sh @@ -22,6 +22,12 @@ UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/lin # Note: UPSTREAM_GIT_BRANCH is not used here UPSTREAM_GIT_BRANCH="none" +# +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version. +# +PACKAGE_DEPENDENCIES="delphix-rust" + # # Force push required when syncing with upstream because we perform a rebase. # diff --git a/packages/linux-kernel-generic/config.delphix.sh b/packages/linux-kernel-generic/config.delphix.sh index 4a8d4e10..4b61fcd1 100644 --- a/packages/linux-kernel-generic/config.delphix.sh +++ b/packages/linux-kernel-generic/config.delphix.sh @@ -22,6 +22,12 @@ UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/ # Note: UPSTREAM_GIT_BRANCH is not used here UPSTREAM_GIT_BRANCH="none" +# +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version. +# +PACKAGE_DEPENDENCIES="delphix-rust" + # # Force push required when syncing with upstream because we perform a rebase. # diff --git a/packages/linux-kernel-oracle/config.delphix.sh b/packages/linux-kernel-oracle/config.delphix.sh index e41c05d4..8d6518b5 100644 --- a/packages/linux-kernel-oracle/config.delphix.sh +++ b/packages/linux-kernel-oracle/config.delphix.sh @@ -22,6 +22,12 @@ UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/lin # Note: UPSTREAM_GIT_BRANCH is not used here UPSTREAM_GIT_BRANCH="none" +# +# Ensure that Delphix's version of the rust toolchain is +# installed rather than Ubuntu's version. +# +PACKAGE_DEPENDENCIES="delphix-rust" + # # Force push required when syncing with upstream because we perform a rebase. # diff --git a/packages/make-jpkg/config.sh b/packages/make-jpkg/config.sh deleted file mode 100644 index 518b4bfc..00000000 --- a/packages/make-jpkg/config.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/make-jpkg.git" - -UPSTREAM_GIT_URL=https://salsa.debian.org/java-team/java-package.git -UPSTREAM_GIT_BRANCH=master - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2a641c7f..963a6c99 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -19,19 +19,15 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" - - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh index 3fc41b3b..36225319 100644 --- a/packages/misc-debs/config.sh +++ b/packages/misc-debs/config.sh @@ -47,10 +47,7 @@ SKIP_COPYRIGHTS_CHECK=true function fetch() { logmust cd "$WORKDIR/artifacts" - local debs=( - # Copied from https://s3.amazonaws.com/packages.treasuredata.com/4/ubuntu/focal/pool/contrib/t/td-agent/td-agent_4.4.2-1_arm64.deb - "td-agent_4.4.2-1_amd64.deb b40c1883c3849e9a7bf67762c9f9a87a6119ad98f1fae64a83d754e1275a379a" - ) + local debs=() local url="http://artifactory.delphix.com/artifactory/linux-pkg/misc-debs" diff --git a/packages/mold/config.sh b/packages/mold/config.sh deleted file mode 100644 index 589ef63a..00000000 --- a/packages/mold/config.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2023 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/mold.git" - -UPSTREAM_GIT_URL=https://salsa.debian.org/pkg-llvm-team/mold.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index 1f72783f..e4b015c7 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,22 +19,20 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish host-jdks" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" logmust install_pkgs \ - "$DEPDIR"/adoptopenjdk/*.deb \ "$DEPDIR"/crypt-blowfish/*.deb \ "$DEPDIR"/host-jdks/*.deb } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" export LANG LANG=en_US.UTF-8 diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 6d9e0aed..9fe0ce33 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -33,6 +33,7 @@ function prepare() { devscripts \ dh-autoreconf \ dh-python \ + dh-dkms \ dkms \ fakeroot \ gawk \ @@ -40,12 +41,14 @@ function prepare() { libattr1-dev \ libblkid-dev \ libelf-dev \ + libpam0g \ libselinux-dev \ libselinux1-dev \ libssl-dev \ + libtirpc-dev \ libtool \ libudev-dev \ - llvm-12 \ + llvm-14 \ lsb-release \ lsscsi \ parted \ @@ -56,7 +59,7 @@ function prepare() { zlib1g-dev logmust install_kernel_headers logmust install_pkgs "$DEPDIR"/delphix-rust/*.deb - logmust cargo install cargo-bundle-licenses + logmust cargo install cargo-bundle-licenses@2.1.1 --locked logmust install_pkgs "$DEPDIR"/delphix-go/*.deb logmust install_pkgs "$DEPDIR"/dwarves/*.deb } diff --git a/resources/delphix_kernel_annotations b/resources/delphix_kernel_annotations index d5b4fef1..bcbdb0af 100644 --- a/resources/delphix_kernel_annotations +++ b/resources/delphix_kernel_annotations @@ -104,7 +104,6 @@ CONFIG_EXFAT_FS policy<{'amd64': 'n', 'arm64': ' CONFIG_EXT2_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_EXT3_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_F2FS_FS policy<{'amd64': 'n', 'arm64': 'n'}> -CONFIG_FAT_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_GFS2_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_HFSPLUS_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_HFS_FS policy<{'amd64': 'n', 'arm64': 'n'}> @@ -126,7 +125,6 @@ CONFIG_SYSV_FS policy<{'amd64': 'n', 'arm64': ' CONFIG_UBIFS_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_UDF_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_UFS_FS policy<{'amd64': 'n', 'arm64': 'n'}> -CONFIG_VFAT_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_VXFS_FS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_XFS_FS policy<{'amd64': 'n', 'arm64': 'n'}> diff --git a/setup.sh b/setup.sh index 9be7405f..5c579604 100755 --- a/setup.sh +++ b/setup.sh @@ -19,7 +19,7 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "$TOP/lib/common.sh" logmust check_running_system -check_env DEFAULT_GIT_BRANCH +check_env DEFAULT_GIT_BRANCH DELPHIX_RELEASE_VERSION # # Update the sources.list file to point to our internal package mirror. If no @@ -31,7 +31,14 @@ function configure_apt_sources() { local secondary_url="$DELPHIX_PACKAGE_MIRROR_SECONDARY" if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then - local latest_url="http://linux-package-mirror.delphix.com/" + local latest_url + local delphix_version="$DELPHIX_RELEASE_VERSION" + if compare_versions "$delphix_version" eq "9999.0.0.0" || compare_versions "$delphix_version" gt "2025.3"; then + latest_url="http://linux-package-mirror-v2.delphix.com/" + else + latest_url="http://linux-package-mirror.delphix.com/" + fi + if is_release_branch; then package_mirror_url="${latest_url}releases/${DELPHIX_RELEASE_VERSION}" else @@ -74,7 +81,6 @@ function configure_apt_sources() { deb-src ${primary_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse deb ${secondary_url} ${UBUNTU_DISTRIBUTION} main multiverse universe - deb ${secondary_url} ${UBUNTU_DISTRIBUTION}-updates main multiverse universe EOF" || die "/etc/apt/sources.list could not be updated" logmust sudo apt-key add "$TOP/resources/delphix-secondary-mirror.key" From 255641ca501095e7958e53f0a3c6004e999c545a Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 12 May 2025 09:10:25 -0700 Subject: [PATCH 2/2] fixup: remove unnecessary prepare functions --- packages/containerized-masking/config.sh | 4 ---- packages/delphix-sso-app/config.sh | 4 ---- packages/drgn/config.sh | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 7b2fd926..31776f51 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -31,10 +31,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" SKIP_COPYRIGHTS_CHECK=true -function prepare() { - echo "Nothing to prepare" -} - function build() { export JAVA_HOME JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 79ac9139..68932535 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -18,10 +18,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" -function prepare() { - echo "Nothing to prepare" -} - function build() { local java_home java_home="/usr/lib/jvm/java-8-openjdk-amd64/" diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 4a180ca1..0387a3a7 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2019, 2025 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.