curio migration command

Author: LexLuthr

.github/image/Dockerfile

@@ -18,12 +18,12 @@ RUN git submodule update --init
 RUN go mod download
 
 # Download Go dependencies for Curio
-WORKDIR /app/boost
+WORKDIR /app/curio
 RUN git submodule update --init
 RUN go mod download
 
 # Stage 2: Install Lotus binary
-FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.32.1-devnet AS lotus-test
+FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.32.2-devnet AS lotus-test
 
 # Stage 3: Build the final image
 FROM myoung34/github-runner AS curio-github-runner

.gitignore

@@ -8,6 +8,7 @@
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
+/migrate-curio
 extern/filecoin-ffi/rust/target
 extern/boostd-data/boostd-data
 **/*.a

Makefile

@@ -133,6 +133,11 @@ boostci: $(BUILD_DEPS)
 	$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
 .PHONY: boostci
 
+migrate-curio: $(BUILD_DEPS)
+	rm -f migrate-curio
+	$(GOCC) build $(GOFLAGS) -o migrate-curio ./cmd/migrate-curio
+.PHONY: boostci
+
 react: validate-node-version
 	npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
 	npm run --prefix react build
@@ -143,7 +148,7 @@ update-react: validate-node-version
 	npm run --prefix react build
 .PHONY: react
 
-build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid migrate-curio
 .PHONY: build-go
 
 build: react build-go
@@ -162,6 +167,7 @@ install-boost:
 	install -C ./booster-http /usr/local/bin/booster-http
 	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
 	install -C ./migrate-lid /usr/local/bin/migrate-lid
+	install -C ./migrate-curio /usr/local/bin/migrate-curio
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet
@@ -217,7 +223,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin
 
 ## DOCKER IMAGES
 docker_user?=filecoin
-lotus_version?=v1.32.1
+lotus_version?=v1.32.2
 ffi_from_source?=0
 build_lotus?=0
 ifeq ($(build_lotus),1)

api/api.go

@@ -26,6 +26,7 @@ type Boost interface {
 	Net
 
 	// MethodGroup: Boost
+	BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error)                                                                               //perm:admin
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin

api/proxy_gen.go

@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
 		indexProvAnnounceLatestHttp,
 		indexProvAnnounceDealRemovalAd,
 		indexProvAnnounceDeal,
+		indexProvRemoveAllCmd,
 	},
 }
 
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
 		return nil
 	},
 }
+
+var indexProvRemoveAllCmd = &cli.Command{
+	Name:  "remove-all",
+	Usage: "Announce all removal ad for all contextIDs",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// announce markets and boost deals
+		cids, err := napi.BoostIndexerRemoveAll(ctx)
+		if err != nil {
+			return err
+		}
+		for _, c := range cids {
+			fmt.Println("Published the removal ad with CID", c.String())
+		}
+		return nil
+	},
+}

build/openrpc/boost.json.gz

@@ -13,6 +13,7 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipld/go-ipld-prime/traversal"
+	nilrouting "github.com/libp2p/go-libp2p-routing-helpers"
 	"github.com/libp2p/go-libp2p/core/host"
 	mh "github.com/multiformats/go-multihash"
 
@@ -21,7 +22,6 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/boxo/bitswap/client"
 	bsnetwork "github.com/ipfs/boxo/bitswap/network"
-	nilrouting "github.com/ipfs/boxo/routing/none"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ipldlegacy "github.com/ipfs/go-ipld-legacy"
@@ -96,10 +96,7 @@ var fetchCmd = &cli.Command{
 		}
 
 		// Create a bitswap client
-		nilRouter, err := nilrouting.ConstructNilRouting(ctx, nil, nil, nil)
-		if err != nil {
-			return err
-		}
+		nilRouter := nilrouting.Null{}
 		net := bsnetwork.NewFromIpfsHost(host, nilRouter)
 		bs, err := blockstore.OpenReadWrite(outputCarPath, []cid.Cid{rootCid}, blockstore.UseWholeCIDs(true))
 		if err != nil {

cmd/boostd/index.go

@@ -8,9 +8,9 @@ import (
 	"github.com/filecoin-project/boost/protocolproxy"
 	bsnetwork "github.com/ipfs/boxo/bitswap/network"
 	"github.com/ipfs/boxo/bitswap/server"
-	blockstore "github.com/ipfs/boxo/blockstore"
-	nilrouting "github.com/ipfs/boxo/routing/none"
+	"github.com/ipfs/boxo/blockstore"
 	"github.com/ipfs/go-cid"
+	nilrouting "github.com/libp2p/go-libp2p-routing-helpers"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
@@ -66,10 +66,7 @@ func (s *BitswapServer) Start(ctx context.Context, proxy *peer.AddrInfo, opts *B
 	}
 
 	// Start a bitswap server on the provider
-	nilRouter, err := nilrouting.ConstructNilRouting(s.ctx, nil, nil, nil)
-	if err != nil {
-		return err
-	}
+	nilRouter := nilrouting.Null{}
 	bsopts := []server.Option{
 		server.EngineBlockstoreWorkerCount(opts.EngineBlockstoreWorkerCount),
 		server.EngineTaskWorkerCount(opts.EngineTaskWorkerCount),

cmd/booster-bitswap/client.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cleanupLevelDBCmd = &cli.Command{
+	Name:        "leveldb",
+	Description: "Removes the indexes and other metadata leveldb based LID store",
+	Usage:       "migrate-curio cleanup leveldb",
+	Before:      before,
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("Please remove the directory called 'LID' in the boost repo path to remove leveldb based LID")
+		fmt.Println("This directory can also be present outside of Boost repo if 'boostd-data' was running with a custom repo path")
+		return nil
+	},
+}

cmd/booster-bitswap/server.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"os"
+
+	"github.com/filecoin-project/boost/build"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("migrate-curio")
+
+const (
+	FlagBoostRepo = "boost-repo"
+)
+
+var FlagRepo = &cli.StringFlag{
+	Name:    FlagBoostRepo,
+	EnvVars: []string{"BOOST_PATH"},
+	Usage:   "boost repo path",
+	Value:   "~/.boost",
+}
+
+var IsVeryVerbose bool
+
+var FlagVeryVerbose = &cli.BoolFlag{
+	Name:        "vv",
+	Usage:       "enables very verbose mode, useful for debugging the CLI",
+	Destination: &IsVeryVerbose,
+}
+
+func main() {
+	app := &cli.App{
+		Name:                 "migrate-curio",
+		Usage:                "Migrate boost to Curio",
+		EnableBashCompletion: true,
+		Version:              build.UserVersion(),
+		Flags: []cli.Flag{
+			FlagRepo,
+			FlagVeryVerbose,
+			&cli.StringFlag{
+				Name:    "db-host",
+				EnvVars: []string{"CURIO_DB_HOST", "CURIO_HARMONYDB_HOSTS"},
+				Usage:   "Command separated list of hostnames for yugabyte cluster",
+				Value:   "127.0.0.1",
+			},
+			&cli.StringFlag{
+				Name:    "db-name",
+				EnvVars: []string{"CURIO_DB_NAME", "CURIO_HARMONYDB_NAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-user",
+				EnvVars: []string{"CURIO_DB_USER", "CURIO_HARMONYDB_USERNAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-password",
+				EnvVars: []string{"CURIO_DB_PASSWORD", "CURIO_HARMONYDB_PASSWORD"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-port",
+				EnvVars: []string{"CURIO_DB_PORT", "CURIO_HARMONYDB_PORT"},
+				Value:   "5433",
+			},
+		},
+		Commands: []*cli.Command{
+			migrateCmd,
+			cleanupLIDCmd,
+		},
+	}
+	app.Setup()
+
+	if err := app.Run(os.Args); err != nil {
+		os.Stderr.WriteString("Error: " + err.Error() + "\n")
+	}
+}
+
+func before(cctx *cli.Context) error {
+	_ = logging.SetLogLevel("migrate-curio", "INFO")
+
+	if IsVeryVerbose {
+		_ = logging.SetLogLevel("migrate-curio", "DEBUG")
+	}
+
+	return nil
+}