[tmpnet] Enable deployment to kube

Author: maru-ava

.github/workflows/ci.yml

@@ -84,6 +84,37 @@ jobs:
           prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
           loki_username: ${{ secrets.LOKI_ID || '' }}
           loki_password: ${{ secrets.LOKI_PASSWORD || '' }}
+  e2e_kube:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-go-for-project
+      # TODO(marun) Package this as a custom action for reuse
+      - uses: ./.github/actions/install-nix
+      - name: Run e2e tests
+        shell: bash
+        run: nix develop --command bash -x ./scripts/tests.e2e.kube.sh
+        env:
+          PROMETHEUS_USERNAME: ${{ secrets.PROMETHEUS_ID || '' }}
+          PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
+          LOKI_USERNAME: ${{ secrets.LOKI_ID || '' }}
+          LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD || '' }}
+          GH_REPO: ${{ github.repository_owner }}/${{ github.event.repository.name }}
+          GH_WORKFLOW: ${{ github.workflow }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_NUMBER: ${{ github.run_number }}
+          GH_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GH_JOB_ID: ${{ github.job }}
+      - name: Export kind logs
+        if: always()
+        shell: bash
+        run: kind export logs /tmp/kind-logs
+      - name: Upload kind logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: kind-logs
+          path: /tmp/kind-logs
+          if-no-files-found: error
   e2e_existing_network:
     runs-on: ubuntu-latest
     steps:
@@ -252,7 +283,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
       - uses: ./.github/actions/install-nix
-      - run: nix develop --command echo "dependencies installed"
       - name: Run e2e tests
         shell: bash
         run: nix develop --command bash -x ./scripts/tests.e2e.bootstrap_monitor.sh

scripts/build_antithesis_images.sh

@@ -87,5 +87,5 @@ else
                                 "${AVALANCHE_PATH}/build/antithesis/xsvm" \
                                 "AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVAGO_PLUGIN_DIR=${AVALANCHE_PATH}/build/plugins"
 
-  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
+  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/Dockerfile.node"
 fi

scripts/build_xsvm_image.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if ! [[ "$0" =~ scripts/build_xsvm_image.sh ]]; then
+  echo "must be run from repository root"
+  exit 255
+fi
+
+# Directory above this script
+AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
+
+# TODO(marun) This image name should be configurable
+DOCKER_IMAGE="localhost:5001/avalanchego"
+
+# Build the avalancehgo node image
+FORCE_TAG_LATEST=1 SKIP_BUILD_RACE=1 DOCKER_IMAGE="${DOCKER_IMAGE}" ./scripts/build_image.sh
+
+# TODO(marun) conditionally push the image to the registry
+GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
+docker buildx build --build-arg GO_VERSION="${GO_VERSION}" --build-arg AVALANCHEGO_NODE_IMAGE="${DOCKER_IMAGE}" \
+       --push -t "${DOCKER_IMAGE}-xsvm" -f "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile" .

scripts/tests.e2e.bootstrap_monitor.sh

@@ -9,6 +9,8 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
   exit 255
 fi
 
+export KUBECONFIG="$HOME/.kube/config"
+
 ./bin/tmpnetctl start-kind-cluster
 
-KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e
+./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e

scripts/tests.e2e.kube.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Run e2e tests against nodes deployed to a kind cluster.
+
+# TODO(marun)
+# - Support testing against a remote cluster
+
+if ! [[ "$0" =~ scripts/tests.e2e.kube.sh ]]; then
+  echo "must be run from repository root"
+  exit 255
+fi
+
+export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
+
+./bin/tmpnetctl start-kind-cluster
+
+if [[ -z "${SKIP_BUILD_IMAGE:-}" ]]; then
+  bash -x ./scripts/build_xsvm_image.sh
+fi
+
+E2E_SERIAL=1 PATH="${PWD}/bin:$PATH" bash -x ./scripts/tests.e2e.sh --runtime=kube --image-name=localhost:5001/avalanchego-xsvm

scripts/tests.e2e.sh

@@ -20,14 +20,17 @@ fi
 # the instructions to build non-portable BLST.
 source ./scripts/constants.sh
 
-# Enable subnet testing by building xsvm
-./scripts/build_xsvm.sh
-echo ""
-
 # Ensure an absolute path to avoid dependency on the working directory
 # of script execution.
-AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
-E2E_ARGS="--avalanchego-path=${AVALANCHEGO_PATH}"
+E2E_ARGS="${*:-}"
+if ! [[ "${E2E_ARGS}" =~ "--runtime=kube" ]]; then
+  # If not running in kubernetes, use the local avalanchego binary
+  AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
+  E2E_ARGS+=" --avalanchego-path=${AVALANCHEGO_PATH}"
+
+  # Enable subnet testing by building the xsvm binary
+  ./scripts/build_xsvm.sh
+fi
 
 #################################
 # Determine ginkgo args
@@ -58,5 +61,5 @@ else
 fi
 
 #################################
-# shellcheck disable=SC2086
-./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}"
+# shellcheck disable=SC2086,SC2068
+./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- ${E2E_ARGS[@]}

tests/antithesis/compose.go

@@ -38,6 +38,12 @@ var (
 // simplify usage by main entrypoints. If the provided network includes a subnet, the initial DB state for
 // the subnet will be created and written to the target path.
 func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error {
+	// TODO(marun) Is there a better way to ensure parity between the configuration that initializes the database and the configuration used at runtime?
+	if network.DefaultFlags == nil {
+		network.DefaultFlags = tmpnet.FlagsMap{}
+	}
+	network.DefaultFlags.SetDefaults(tmpnet.DefaultTestFlags())
+
 	targetPath := os.Getenv(targetPathEnvName)
 	if len(targetPath) == 0 {
 		return errTargetPathEnvVarNotSet
@@ -66,7 +72,16 @@ func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error
 			return fmt.Errorf("failed to get bootstrap volume path: %w", err)
 		}
 
-		if err := initBootstrapDB(network, avalancheGoPath, pluginDir, bootstrapVolumePath); err != nil {
+		network.DefaultRuntimeConfig = tmpnet.NodeRuntimeConfig{
+			AvalancheGoPath: avalancheGoPath,
+		}
+		// TODO(marun) Need to have a standard way of initializing a network
+		if network.DefaultFlags == nil {
+			network.DefaultFlags = tmpnet.FlagsMap{}
+		}
+		network.DefaultFlags[config.PluginDirKey] = pluginDir
+
+		if err := initBootstrapDB(network, bootstrapVolumePath); err != nil {
 			return fmt.Errorf("failed to initialize db volumes: %w", err)
 		}
 	}

tests/antithesis/init_db.go

@@ -28,16 +28,14 @@ func getBootstrapVolumePath(targetPath string) (string, error) {
 // Bootstraps a local process-based network, creates its subnets and chains, and copies
 // the resulting db state from one of the nodes to the provided path. The path will be
 // created if it does not already exist.
-func initBootstrapDB(network *tmpnet.Network, avalancheGoPath string, pluginDir string, destPath string) error {
+func initBootstrapDB(network *tmpnet.Network, destPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
 	defer cancel()
 	if err := tmpnet.BootstrapNewNetwork(
 		ctx,
 		tests.NewDefaultLogger(""),
 		network,
 		"",
-		avalancheGoPath,
-		pluginDir,
 	); err != nil {
 		return fmt.Errorf("failed to bootstrap network: %w", err)
 	}

tests/e2e/c/dynamic_fees.go

@@ -60,7 +60,7 @@ var _ = e2e.DescribeCChain("[Dynamic Fees]", func() {
 		env := e2e.GetEnv(tc)
 		publicNetwork := env.GetNetwork()
 
-		privateNetwork := tmpnet.NewDefaultNetwork("avalanchego-e2e-dynamic-fees")
+		privateNetwork := tmpnet.NewDefaultNetwork(tc.Log(), "avalanchego-e2e-dynamic-fees")
 		// Copy over the defaults from the normal test suite to include settings
 		// like the upgrade config.
 		privateNetwork.DefaultFlags = tmpnet.FlagsMap{}

tests/e2e/e2e_test.go

@@ -75,5 +75,5 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// Run in every ginkgo process
 
 	// Initialize the local test environment from the global state
-	e2e.InitSharedTestEnvironment(ginkgo.GinkgoT(), envBytes)
+	e2e.InitSharedTestEnvironment(e2e.NewTestContext(), envBytes)
 })

tests/e2e/faultinjection/duplicate_node_id.go

@@ -26,6 +26,11 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
 	ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
 		network := e2e.GetEnv(tc).GetNetwork()
 
+		if network.DefaultRuntimeConfig.KubeRuntimeConfig != nil {
+			// Enabling this test for kube requires supporting a flexible name mapping
+			ginkgo.Skip("This test is not supported on kube to avoid having to deviate from composing the statefulset name with the network uuid + nodeid")
+		}
+
 		tc.By("creating new node")
 		node1 := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
 		e2e.WaitForHealthy(tc, node1)
@@ -43,10 +48,10 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
 			// the same node ID.
 			config.DataDirKey: fmt.Sprintf("%s-second", node1Flags[config.DataDirKey]),
 		}
-		node2 := e2e.AddEphemeralNode(tc, network, node2Flags)
+		node2 := e2e.AddEphemeralNodeWithWaitForHealth(tc, network, node2Flags, false /* waitForHealth */)
 
 		tc.By("checking that the second new node fails to become healthy before timeout")
-		err := tmpnet.WaitForHealthy(tc.DefaultContext(), node2)
+		err := tmpnet.WaitForHealthyNode(tc.DefaultContext(), tc.Log(), node2)
 		require.ErrorIs(err, context.DeadlineExceeded)
 
 		tc.By("stopping the first new node")
@@ -68,7 +73,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
 	require := require.New(tc)
 
 	// Collect the node ids of the new node's peers
-	infoClient := info.NewClient(newNode.URI)
+	uri := e2e.GetLocalURI(tc, newNode)
+	infoClient := info.NewClient(uri)
 	peers, err := infoClient.Peers(tc.DefaultContext(), nil)
 	require.NoError(err)
 	peerIDs := set.NewSet[ids.NodeID](len(existingNodes))
@@ -81,7 +87,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
 		require.True(peerIDs.Contains(existingNode.NodeID))
 
 		// Check that the new node is a peer
-		infoClient := info.NewClient(existingNode.URI)
+		uri := e2e.GetLocalURI(tc, existingNode)
+		infoClient := info.NewClient(uri)
 		peers, err := infoClient.Peers(tc.DefaultContext(), nil)
 		require.NoError(err)
 		isPeer := false

tests/fixture/bootstrapmonitor/e2e/e2e_test.go

@@ -261,14 +261,16 @@ func buildImage(tc tests.TestContext, imageName string, forceNewHash bool, scrip
 	require.NoError(err, "Image build failed: %s", output)
 }
 
-func newNodeStatefulSet(name string, flags map[string]string) *appsv1.StatefulSet {
+func newNodeStatefulSet(name string, flags tmpnet.FlagsMap) *appsv1.StatefulSet {
 	statefulSet := tmpnet.NewNodeStatefulSet(
 		name,
+		true, /* generateName */
 		latestAvalanchegoImage,
 		nodeContainerName,
 		volumeName,
 		volumeSize,
 		nodeDataDir,
+		nil,
 		flags,
 	)
 
@@ -281,8 +283,9 @@ func newNodeStatefulSet(name string, flags map[string]string) *appsv1.StatefulSe
 	return statefulSet
 }
 
-func defaultPodFlags() map[string]string {
-	return tmpnet.DefaultPodFlags(constants.LocalName, nodeDataDir)
+func defaultPodFlags() tmpnet.FlagsMap {
+	// TODO(marun) DefaultPodFlags is only used by bootstrap monitor - maybe inline
+	return tmpnet.DefaultPodFlags(constants.LocalName, nodeDataDir, false /* sybilProtectionEnabled */)
 }
 
 // waitForPodCondition waits until the specified pod reports the specified condition