diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index c257f729f0..0000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: Publish - -# We have to use gtar on macOS because apple's tar is literally broken. -# Yes, I know how stupid that sounds. But it's true: -# https://github.com/actions/virtual-environments/issues/2619 - -on: - push: - tags: - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z]+.[0-9]+' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # When getting Rust dependencies, retry on network error: - CARGO_NET_RETRY: 10 - # Use the local .curlrc - CURL_HOME: . - -jobs: - build_dfx: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # We build a dynamic-linked linux binary because otherwise HSM support fails with: - # Error: IO: Dynamic loading not supported - target: [ x86_64-apple-darwin, x86_64-unknown-linux-gnu ] - include: - - os: macos-12 - target: x86_64-apple-darwin - binary_path: target/x86_64-apple-darwin/release - name: x86_64-darwin - tar: gtar - - os: ubuntu-20.04 - target: x86_64-unknown-linux-gnu - binary_path: target/x86_64-unknown-linux-gnu/release - name: x86_64-linux - tar: tar - steps: - - uses: actions/checkout@v4 - - - name: Setup environment variables - run: | - echo "RUSTFLAGS=--remap-path-prefix=${GITHUB_WORKSPACE}=/builds/dfinity" >> $GITHUB_ENV - - # GITHUB_REF_NAME will be something link 2353/merge for branch builds, which isn't great as a dfx version - - name: Set dfx version (tag builds only) - if: github.ref_type == 'tag' - run: | - echo "DFX_VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV - echo "TARBALL_1_FILENAME=dfx-$GITHUB_REF_NAME-${{ matrix.name }}.tar.gz" >> $GITHUB_ENV - echo "SHA256_1_FILENAME=dfx-$GITHUB_REF_NAME-${{ matrix.name }}.tar.gz.sha256" >> $GITHUB_ENV - echo "TARBALL_2_FILENAME=dfx-${{ matrix.target }}.tar.gz" >> $GITHUB_ENV - echo "SHA256_2_FILENAME=dfx-${{ matrix.target }}.tar.gz.sha256" >> $GITHUB_ENV - - - name: Cache Cargo - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('rust-toolchain.toml') }}-publish-1 - - name: Install Rust - run: rustup show - if: contains(matrix.os, 'macos') - - - name: Build - run: | - cargo clean --target ${{ matrix.target }} --release - cargo build --target ${{ matrix.target }} --locked --release - - - name: Check dynamically-linked libraries (macos) - run: | - ACTUAL="$(otool -L ${{ matrix.binary_path }}/dfx | awk 'NR > 1{ print $1 }' | grep -v /System/Library/Frameworks | sort | awk -v d=" " '{s=(NR==1?s:s d)$0}END{printf "%s",s}')" - EXPECTED="/usr/lib/libSystem.B.dylib /usr/lib/libc++.1.dylib /usr/lib/libiconv.2.dylib" - echo "Dynamically-linked libraries:" - echo " Actual: $ACTUAL" - echo " Expected: $EXPECTED" - if [ "$ACTUAL" != "$EXPECTED" ]; then - exit 1 - fi - if: contains(matrix.os, 'macos') - - - name: Check dynamically-linked libraries (ubuntu) - run: | - ACTUAL="$(ldd ${{ matrix.binary_path }}/dfx | awk '{ print $1 }' | sort | awk -v d=" " '{s=(NR==1?s:s d)$0}END{printf "%s",s}')" - EXPECTED="/lib64/ld-linux-x86-64.so.2 libc.so.6 libdl.so.2 libgcc_s.so.1 libm.so.6 libpthread.so.0 libstdc++.so.6 linux-vdso.so.1" - echo "Dynamically-linked libraries:" - echo " Actual: $ACTUAL" - echo " Expected: $EXPECTED" - if [ "$ACTUAL" != "$EXPECTED" ]; then - exit 1 - fi - if: contains(matrix.os, 'ubuntu') - - - name: Strip binaries - run: | - cd ${{ matrix.binary_path }} - sudo chown -R $(whoami) . - strip dfx - if: contains(matrix.os, 'ubuntu') - - - name: Create tarball of binaries and sha256 of tarball - if: github.ref_type == 'tag' - run: | - mkdir dfx-${{ matrix.target }} - cp ${{ matrix.binary_path }}/dfx dfx-${{ matrix.target }} - cp LICENSE dfx-${{ matrix.target }} - ${{ matrix.tar }} -zc -f ${{ env.TARBALL_2_FILENAME }} dfx-${{ matrix.target }} - shasum -a 256 ${{ env.TARBALL_2_FILENAME }} > ${{ env.SHA256_2_FILENAME }} - shasum -c ${{ env.SHA256_2_FILENAME }} - - ${{ matrix.tar }} -zcC ${{ matrix.binary_path }} -f ${{ env.TARBALL_1_FILENAME }} dfx - shasum -a 256 ${{ env.TARBALL_1_FILENAME }} > $SHA256_1_FILENAME - shasum -c $SHA256_1_FILENAME - - - name: Upload Artifacts - if: github.ref_type == 'tag' - uses: actions/upload-artifact@v4 - with: - name: dfx-artifacts-${{ hashFiles('rust-toolchain.toml') }}-${{ matrix.name }} - path: | - ${{ env.TARBALL_1_FILENAME }} - ${{ env.SHA256_1_FILENAME }} - ${{ env.TARBALL_2_FILENAME }} - ${{ env.SHA256_2_FILENAME }} - - aggregate: - name: publishable:required - if: ${{ always() }} - needs: [build_dfx] - runs-on: ubuntu-latest - steps: - - name: check build result - if: ${{ needs.build_dfx.result != 'success' }} - run: exit 1 - - publish: - runs-on: ubuntu-latest - if: github.ref_type == 'tag' - needs: build_dfx - strategy: - fail-fast: false - matrix: - name: [ 'x86_64-darwin', 'x86_64-linux' ] - steps: - - uses: actions/checkout@v4 - - - name: Setup environment variables - run: echo "VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV - - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - name: dfx-artifacts-${{ hashFiles('rust-toolchain.toml') }}-${{ matrix.name }} - - - name: Upload tarball and sha256 - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: dfx-*.tar.* - file_glob: true - tag: ${{ env.VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..e696ba914e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,256 @@ +# Copyright 2022-2023, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with cargo-dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a Github Release +# +# Note that the Github Release will be created with a generated +# title/body based on your changelogs. + +name: Release + +permissions: + contents: write + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (cargo-dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However Github +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + pull_request: + +jobs: + # Run 'cargo dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: ubuntu-latest + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh" + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + cargo dist ${{ !github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name) || (github.event.pull_request.head.repo.fork && 'plan' || 'host --steps=check') }} --output-format=json > dist-manifest.json + echo "cargo dist ran successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by cargo-dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to cargo dist + # - install-dist: expression to run to install cargo-dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: swatinem/rust-cache@v2 + - name: Install cargo-dist + run: ${{ matrix.install_dist }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "cargo dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh" + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + - id: cargo-dist + shell: bash + run: | + cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "cargo dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh" + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + # This is a harmless no-op for Github Releases, hosting for that happens in "announce" + - id: host + shell: bash + run: | + cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: dist-manifest.json + + # Create a Github Release while uploading all files to it + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: "Download Github Artifacts" + uses: actions/download-artifact@v3 + with: + name: artifacts + path: artifacts + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create Github Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ needs.plan.outputs.tag }} + name: ${{ fromJson(needs.host.outputs.val).announcement_title }} + body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }} + prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }} + artifacts: "artifacts/*" diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a93a44c7b..1e40b54920 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ # UNRELEASED +## Dependencies + +### Frontend canister + +- Module hash: 192b6af80648aa937ca54eda1ea83677bab13f95589cd5bf251043a7a249083f +- https://github.com/dfinity/sdk/pull/3496 + # 0.17.0 ### feat: new starter templates @@ -59,6 +66,7 @@ It is natural to point `wasm_hash_url` to the `.sha256` file generated by Now when `dfx deps pull`, such content will be accept properly. + ### feat: dfx upgrade will direct the user to install dfxvm if it has been released. If the latest release of https://github.com/dfinity/dfxvm is >= 1.0, `dfx upgrade` will @@ -94,7 +102,6 @@ The dfxvm install script now accepts `DFXVM_INIT_YES=` to skip ### chore: bump `ic-agent`, `ic-utils` and `ic-identity-hsm` to 0.32.0 - # 0.16.1 ### feat: query stats support diff --git a/Cargo.toml b/Cargo.toml index bdfba1b241..255753a532 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ edition = "2021" repository = "https://github.com/dfinity/sdk" rust-version = "1.75.0" license = "Apache-2.0" +publish = false [workspace.dependencies] candid = "0.10.3" @@ -73,6 +74,23 @@ tokio = "1.35" url = "2.1.0" walkdir = "2.3.2" +# Config for 'cargo dist' +[workspace.metadata.dist] +# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.5.0" +# CI backends to support +ci = ["github"] +# The installers to generate for each app +installers = [] +# Target platforms to build apps for (Rust target-triple syntax) +targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"] +# Publish jobs to run in CI +pr-run-mode = "upload" +# Build only the required packages, and individually +precise-builds = true +# The archive format to use for non-windows builds (defaults .tar.xz) +unix-archive = ".tar.gz" + [profile.release] panic = 'abort' lto = true @@ -80,5 +98,9 @@ lto = true [profile.dev.package.argon2] opt-level = 3 +# The profile that 'cargo dist' will build with +[profile.dist] +inherits = "release" + [profile.release.package.ic-frontend-canister] opt-level = 'z' diff --git a/src/canisters/frontend/ic-asset/Cargo.toml b/src/canisters/frontend/ic-asset/Cargo.toml index 3139c6edda..09d0fc23d8 100644 --- a/src/canisters/frontend/ic-asset/Cargo.toml +++ b/src/canisters/frontend/ic-asset/Cargo.toml @@ -10,6 +10,7 @@ description = "Library for storing files in an asset canister." documentation = "https://docs.rs/ic-asset" categories = ["api-bindings", "data-structures"] keywords = ["internet-computer", "assets", "icp", "dfinity"] +publish = false [dependencies] backoff.workspace = true diff --git a/src/canisters/frontend/ic-certified-assets/Cargo.toml b/src/canisters/frontend/ic-certified-assets/Cargo.toml index 7f5028204a..9d80b99f7d 100644 --- a/src/canisters/frontend/ic-certified-assets/Cargo.toml +++ b/src/canisters/frontend/ic-certified-assets/Cargo.toml @@ -10,6 +10,7 @@ description = "Rust support for asset certification." documentation = "https://docs.rs/ic-certified-assets" categories = ["wasm", "filesystem", "data-structures"] keywords = ["internet-computer", "dfinity"] +publish = false [dependencies] base64.workspace = true diff --git a/src/canisters/frontend/ic-frontend-canister/Cargo.toml b/src/canisters/frontend/ic-frontend-canister/Cargo.toml index cc1119adac..8d1d7e6bef 100644 --- a/src/canisters/frontend/ic-frontend-canister/Cargo.toml +++ b/src/canisters/frontend/ic-frontend-canister/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/dfinity/sdk" license = "Apache-2.0" categories = ["wasm"] keywords = ["internet-computer", "dfinity"] +publish = false [lib] path = "src/lib.rs" @@ -17,3 +18,7 @@ crate-type = ["cdylib"] ic-certified-assets = { path = "../ic-certified-assets" } ic-cdk.workspace = true candid.workspace = true + +# Config for 'cargo dist' +[package.metadata.dist] +dist = false diff --git a/src/canisters/frontend/icx-asset/Cargo.toml b/src/canisters/frontend/icx-asset/Cargo.toml index 8e1bf0859f..d741397bbc 100644 --- a/src/canisters/frontend/icx-asset/Cargo.toml +++ b/src/canisters/frontend/icx-asset/Cargo.toml @@ -10,6 +10,7 @@ description = "CLI tool to manage assets on an asset canister on the Internet Co documentation = "https://docs.rs/icx-asset" categories = ["command-line-interface"] keywords = ["internet-computer", "agent", "icp", "dfinity", "asset"] +publish = false [dependencies] anstyle.workspace = true diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index fe08ca3087..6175ae3181 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true repository.workspace = true license.workspace = true rust-version.workspace = true +publish = false [dependencies] aes-gcm.workspace = true diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 752ca77d06..289953a402 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true license.workspace = true rust-version.workspace = true build = "assets/build.rs" +publish = false [[bin]] name = "dfx" @@ -125,3 +126,7 @@ env_logger = "0.10" proptest = "1.0" mockito = "0.31.0" tempfile = "3.1.0" + +# Config for 'cargo dist' +[package.metadata.dist] +dist = true diff --git a/src/distributed/assetstorage.wasm.gz b/src/distributed/assetstorage.wasm.gz index 2796c34f49..bb490bd70f 100755 Binary files a/src/distributed/assetstorage.wasm.gz and b/src/distributed/assetstorage.wasm.gz differ