From e11df10eb6903fadca8630476f0ff408340c4d54 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Mar 2021 14:34:26 +0100 Subject: [PATCH 1/8] cmd/geth: iter over snapshot in compute commitments cmd cmd/geth: fix typo cmd/geth: fix iter errors cmd/geth: run go fmt --- cmd/geth/snapshot.go | 58 +++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 5d8d7b0d4f68..1e693a8f360b 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -31,10 +31,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "gopkg.in/urfave/cli.v1" "github.com/gballet/go-verkle" - "github.com/protolambda/go-kzg/bls" "github.com/protolambda/go-kzg" + "github.com/protolambda/go-kzg/bls" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -511,44 +511,58 @@ func computeCommitment(ctx *cli.Context) error { root = head.Root() log.Info("Start traversing the state", "root", root, "number", head.NumberU64()) } + triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := snapshot.New(chaindb, triedb, 256, chain.CurrentBlock().Root(), false, false, false) if err != nil { - log.Error("Failed to open trie", "root", root, "error", err) + log.Error("Failed to open snapshot tree", "error", err) return err } + var ( - accounts int + accounts int //slots int //codes int lastReport time.Time start = time.Now() ) vRoot := verkle.New() - accIter := trie.NewIterator(t.NodeIterator(nil)) + + accIter, err := t.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer accIter.Release() + for accIter.Next() { accounts += 1 - vRoot.InsertOrdered(accIter.Key, accIter.Value, ks, lg1) - - var acc state.Account - if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { - log.Error("Invalid account encountered during traversal", "error", err) + key := accIter.Hash().Bytes() + slimAcc := accIter.Account() + acc, err := snapshot.FullAccount(slimAcc) + if err != nil { + log.Error("Failed parsing slim account from snapshot", "error", err) return err } - if acc.Root != emptyRoot { + value, err := rlp.EncodeToBytes(acc) + if err != nil { + log.Error("Failed to encode account to rlp", "error", err) + } + vRoot.InsertOrdered(key, value, ks, lg1) + + if !bytes.Equal(acc.Root, emptyRoot.Bytes()) { sRoot := verkle.New() - storageTrie, err := trie.NewSecure(acc.Root, triedb) + storageIter, err := t.StorageIterator(root, accIter.Hash(), common.Hash{}) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "error", err) return err } - storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) + defer storageIter.Release() + for storageIter.Next() { - sRoot.InsertOrdered(storageIter.Key, storageIter.Value, ks, lg1) + sRoot.InsertOrdered(storageIter.Hash().Bytes(), storageIter.Slot(), ks, lg1) } - if storageIter.Err != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err) - return storageIter.Err + if storageIter.Error() != nil { + log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error()) + return storageIter.Error() } } if time.Since(lastReport) > time.Second*8 { @@ -556,9 +570,9 @@ func computeCommitment(ctx *cli.Context) error { lastReport = time.Now() } } - if accIter.Err != nil { - log.Error("Failed to compute commitment", "root", root, "error", accIter.Err) - return accIter.Err + if accIter.Error() != nil { + log.Error("Failed to compute commitment", "root", root, "error", accIter.Error()) + return accIter.Error() } log.Info("Commitment computation complete", "compressed", bls.ToCompressedG1(vRoot.GetCommitment()), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nil From 70e29b0f966d0f4020a651e8ac456d647dc86288 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Mar 2021 15:22:04 +0100 Subject: [PATCH 2/8] cmd/geth: compute commitment after iterating slots & accs --- cmd/geth/snapshot.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1e693a8f360b..dff289765df0 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -564,6 +564,7 @@ func computeCommitment(ctx *cli.Context) error { log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error()) return storageIter.Error() } + sRoot.ComputeCommitment(ks, lg1) } if time.Since(lastReport) > time.Second*8 { log.Info("Traversing state", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) @@ -574,6 +575,7 @@ func computeCommitment(ctx *cli.Context) error { log.Error("Failed to compute commitment", "root", root, "error", accIter.Error()) return accIter.Error() } + vRoot.ComputeCommitment(ks, lg1) log.Info("Commitment computation complete", "compressed", bls.ToCompressedG1(vRoot.GetCommitment()), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 16178fcd300f9b3f5120e27165cb2e8fc6c7086a Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Mar 2021 16:54:29 +0100 Subject: [PATCH 3/8] cmd,core: use existing generateTrieRoot method for compute commitment --- cmd/geth/snapshot.go | 70 ++++++------------------------- core/state/snapshot/conversion.go | 22 +++++----- core/state/snapshot/snapshot.go | 29 +++++++++++++ 3 files changed, 53 insertions(+), 68 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index dff289765df0..37c26a814b32 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -483,6 +484,16 @@ func computeCommitment(ctx *cli.Context) error { } ks := kzg.NewKZGSettings(fftCfg, s1, s2) + verkleGenerate := func(db ethdb.KeyValueWriter, in chan snapshot.TrieKV, out chan common.Hash) { + t := verkle.New() + for leaf := range in { + t.InsertOrdered(leaf.Key[:], leaf.Value, ks, lg1) + } + comm := t.ComputeCommitment(ks, lg1) + root := common.BytesToHash(bls.ToCompressedG1(comm)) + out <- root + } + stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -519,64 +530,9 @@ func computeCommitment(ctx *cli.Context) error { return err } - var ( - accounts int - //slots int - //codes int - lastReport time.Time - start = time.Now() - ) - vRoot := verkle.New() - - accIter, err := t.AccountIterator(root, common.Hash{}) - if err != nil { - return err - } - defer accIter.Release() - - for accIter.Next() { - accounts += 1 - key := accIter.Hash().Bytes() - slimAcc := accIter.Account() - acc, err := snapshot.FullAccount(slimAcc) - if err != nil { - log.Error("Failed parsing slim account from snapshot", "error", err) - return err - } - value, err := rlp.EncodeToBytes(acc) - if err != nil { - log.Error("Failed to encode account to rlp", "error", err) - } - vRoot.InsertOrdered(key, value, ks, lg1) - - if !bytes.Equal(acc.Root, emptyRoot.Bytes()) { - sRoot := verkle.New() - storageIter, err := t.StorageIterator(root, accIter.Hash(), common.Hash{}) - if err != nil { - return err - } - defer storageIter.Release() - - for storageIter.Next() { - sRoot.InsertOrdered(storageIter.Hash().Bytes(), storageIter.Slot(), ks, lg1) - } - if storageIter.Error() != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error()) - return storageIter.Error() - } - sRoot.ComputeCommitment(ks, lg1) - } - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if accIter.Error() != nil { - log.Error("Failed to compute commitment", "root", root, "error", accIter.Error()) - return accIter.Error() + if err := t.ComputeVerkleCommitment(root, verkleGenerate); err != nil { + log.Error("Failed to compute verkle commitment", "error", err) } - vRoot.ComputeCommitment(ks, lg1) - log.Info("Commitment computation complete", "compressed", bls.ToCompressedG1(vRoot.GetCommitment()), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nil } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index bb87ecddf189..f458b15894cd 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -34,16 +34,16 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// trieKV represents a trie key-value pair -type trieKV struct { - key common.Hash - value []byte +// TrieKV represents a trie key-value pair +type TrieKV struct { + Key common.Hash + Value []byte } type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (TrieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -244,7 +244,7 @@ func runReport(stats *generateStats, stop chan bool) { // whole state which connects the accounts and the corresponding storages. func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { var ( - in = make(chan trieKV) // chan to pass leaves + in = make(chan TrieKV) // chan to pass leaves out = make(chan common.Hash, 1) // chan to collect result stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled wg sync.WaitGroup @@ -289,7 +289,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, var ( logged = time.Now() processed = uint64(0) - leaf trieKV + leaf TrieKV ) // Start to feed leaves for it.Next() { @@ -332,9 +332,9 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(err) } } - leaf = trieKV{it.Hash(), fullData} + leaf = TrieKV{it.Hash(), fullData} } else { - leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} + leaf = TrieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} } in <- leaf @@ -360,10 +360,10 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { +func stackTrieGenerate(db ethdb.KeyValueWriter, in chan TrieKV, out chan common.Hash) { t := trie.NewStackTrie(db) for leaf := range in { - t.TryUpdate(leaf.key[:], leaf.value) + t.TryUpdate(leaf.Key[:], leaf.Value) } var root common.Hash if db == nil { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index aa5f5900b0cf..dec5bc490dbe 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -734,6 +734,35 @@ func (t *Tree) Verify(root common.Hash) error { return nil } +// Computes verkle commitment against snapshot +func (t *Tree) ComputeVerkleCommitment(root common.Hash, generatorFn trieGeneratorFn) error { + acctIt, err := t.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer acctIt.Release() + + got, err := generateTrieRoot(nil, acctIt, common.Hash{}, generatorFn, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, err := t.StorageIterator(root, accountHash, common.Hash{}) + if err != nil { + return common.Hash{}, err + } + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, storageIt, accountHash, generatorFn, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + + if err != nil { + return err + } + log.Info("Computed verkle commitment", "commitment", got) + return nil +} + // disklayer is an internal helper function to return the disk layer. // The lock of snapTree is assumed to be held already. func (t *Tree) disklayer() *diskLayer { From 2b7b626c610eac1e0f4287ce7bc1ab217de43db0 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Mar 2021 17:16:26 +0100 Subject: [PATCH 4/8] core/state: add checkSubRoot flag to generateTrieRoot --- core/state/snapshot/conversion.go | 12 ++++++------ core/state/snapshot/snapshot.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index f458b15894cd..d7cbd78b8498 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -52,12 +52,12 @@ type ( // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true, true) } // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true, true) } // GenerateTrie takes the whole snapshot tree as the input, traverses all the @@ -87,12 +87,12 @@ func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethd } defer storageIt.Release() - hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false, true) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, newGenerateStats(), true, true) if err != nil { return err @@ -242,7 +242,7 @@ func runReport(stats *generateStats, stop chan bool) { // generateTrieRoot generates the trie hash based on the snapshot iterator. // It can be used for generating account trie, storage trie or even the // whole state which connects the accounts and the corresponding storages. -func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { +func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool, checkSubRoot bool) (common.Hash, error) { var ( in = make(chan TrieKV) // chan to pass leaves out = make(chan common.Hash, 1) // chan to collect result @@ -321,7 +321,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, results <- err return } - if !bytes.Equal(account.Root, subroot.Bytes()) { + if checkSubRoot && !bytes.Equal(account.Root, subroot.Bytes()) { results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot) return } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index dec5bc490dbe..3d7f8f9d2669 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -718,12 +718,12 @@ func (t *Tree) Verify(root common.Hash) error { } defer storageIt.Release() - hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false, true) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, newGenerateStats(), true, true) if err != nil { return err @@ -749,12 +749,12 @@ func (t *Tree) ComputeVerkleCommitment(root common.Hash, generatorFn trieGenerat } defer storageIt.Release() - hash, err := generateTrieRoot(nil, storageIt, accountHash, generatorFn, nil, stat, false) + hash, err := generateTrieRoot(nil, storageIt, accountHash, generatorFn, nil, stat, false, false) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, newGenerateStats(), true, false) if err != nil { return err From d446cfc3bb04eda9a5933242b258f1fb79c0b724 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 26 Mar 2021 10:20:18 +0100 Subject: [PATCH 5/8] cmd/geth: fix bug in verkleGenerate --- cmd/geth/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 37c26a814b32..fa80f8981562 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -487,7 +487,7 @@ func computeCommitment(ctx *cli.Context) error { verkleGenerate := func(db ethdb.KeyValueWriter, in chan snapshot.TrieKV, out chan common.Hash) { t := verkle.New() for leaf := range in { - t.InsertOrdered(leaf.Key[:], leaf.Value, ks, lg1) + t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1) } comm := t.ComputeCommitment(ks, lg1) root := common.BytesToHash(bls.ToCompressedG1(comm)) From eaad65ed1001bc22fdd4b2da928da559e4c5badf Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 31 Mar 2021 20:33:19 +0200 Subject: [PATCH 6/8] cmd/geth: write verkle nodes to db --- cmd/geth/snapshot.go | 35 +++++++++++++++++++++++++++++------ go.mod | 7 ++++--- go.sum | 8 ++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index fa80f8981562..d4344a728ddf 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -484,21 +484,43 @@ func computeCommitment(ctx *cli.Context) error { } ks := kzg.NewKZGSettings(fftCfg, s1, s2) + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, chaindb := utils.MakeChain(ctx, stack, true) + defer chaindb.Close() + + verkledb, err := stack.OpenDatabase("verkle", 0, 0, "") + if err != nil { + log.Error("Failed to open db for verkle nodes", "error", err) + return err + } + defer verkledb.Close() + + nodesCh := make(chan verkle.FlushableNode) verkleGenerate := func(db ethdb.KeyValueWriter, in chan snapshot.TrieKV, out chan common.Hash) { t := verkle.New() for leaf := range in { - t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1) + t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1, nodesCh) } comm := t.ComputeCommitment(ks, lg1) root := common.BytesToHash(bls.ToCompressedG1(comm)) out <- root } - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chain, chaindb := utils.MakeChain(ctx, stack, true) - defer chaindb.Close() + nodesCount := 0 + go func() { + for fn := range nodesCh { + nodesCount++ + value, err := rlp.EncodeToBytes(fn.Node) + if err != nil { + log.Error("Failed to encode verkle node", "error", err) + } + if err := verkledb.Put(fn.Hash[:], value); err != nil { + log.Error("Failed to write verkle node to db", "error", err) + } + } + }() if ctx.NArg() > 1 { log.Error("Too many arguments given") @@ -533,6 +555,7 @@ func computeCommitment(ctx *cli.Context) error { if err := t.ComputeVerkleCommitment(root, verkleGenerate); err != nil { log.Error("Failed to compute verkle commitment", "error", err) } + log.Info("Number of nodes written to DB: %d\n", nodesCount) return nil } diff --git a/go.mod b/go.mod index e72158b000eb..9f46e9953299 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/fatih/color v1.7.0 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9 + github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 @@ -26,6 +26,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d + github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.1.1 github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031 @@ -49,8 +50,8 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef - golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 - golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index 9bf821500e08..2e6ccdcef93b 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,8 @@ github.com/gballet/go-verkle v0.0.0-20210305125751-d07bfb0dd271 h1:SQ8VtqI0zpGTA github.com/gballet/go-verkle v0.0.0-20210305125751-d07bfb0dd271/go.mod h1:qJML0ZSm9IvJ4SV9Iy/s5mNYld5tyLvVzoHTaC1Weq0= github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9 h1:rOR2V9fSuvpAO+4k6c34OzDyAAlA4eGLSJoYTlxZQns= github.com/gballet/go-verkle v0.0.0-20210318184615-3228314e99c9/go.mod h1:s2tVAxI5FnThDvOCzrUpNU8HxoGvFBelNLZpDEEsFuk= +github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec h1:SFb34albxbnAcY+wLS7nKL+nG6RVcQpmREE2lpJLEIk= +github.com/gballet/go-verkle v0.0.0-20210331172732-b0e741f9a8ec/go.mod h1:s2tVAxI5FnThDvOCzrUpNU8HxoGvFBelNLZpDEEsFuk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= @@ -258,6 +260,8 @@ github.com/herumi/bls-eth-go-binary v0.0.0-20210128192016-d750c7e71022/go.mod h1 github.com/herumi/bls-eth-go-binary v0.0.0-20210302070600-dfaa902c7773/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v0.0.0-20210311024029-2939a1bfdec2 h1:WYYASILSlSdBFLQRysRQFuRW7VHxyNDbmYaGo3T2PkA= github.com/herumi/bls-eth-go-binary v0.0.0-20210311024029-2939a1bfdec2/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743 h1:r9ipCR8S2CsnXftdEBGBRNvd69Hmggm83eajtzsguw4= +github.com/herumi/bls-eth-go-binary v0.0.0-20210325004959-a24496ac0743/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= @@ -496,6 +500,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 h1:gzMM0EjIYiRmJI3+jBdFuoynZlpxa2JQZsolKu09BXo= golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -600,6 +606,8 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEq golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d h1:jbzgAvDZn8aEnytae+4ou0J0GwFZoHR0hOrTg4qH8GA= golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From a50ff564c6d88688a77801eefaf438199082eeac Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 31 Mar 2021 20:37:07 +0200 Subject: [PATCH 7/8] cmd/geth: flush verkle internal node after insertions --- cmd/geth/snapshot.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index d4344a728ddf..39375628a7ca 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -504,6 +504,8 @@ func computeCommitment(ctx *cli.Context) error { t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1, nodesCh) } comm := t.ComputeCommitment(ks, lg1) + // Flush remaining nodes to nodes channel + t.Flush(nodesCh) root := common.BytesToHash(bls.ToCompressedG1(comm)) out <- root } From 54a92313b0010c213561cef84606204994c398fd Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 1 Apr 2021 09:20:00 +0200 Subject: [PATCH 8/8] cmd/geth: use node.Serialize --- cmd/geth/snapshot.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 39375628a7ca..880533c906ac 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -503,9 +503,9 @@ func computeCommitment(ctx *cli.Context) error { for leaf := range in { t.InsertOrdered(common.CopyBytes(leaf.Key[:]), leaf.Value, ks, lg1, nodesCh) } - comm := t.ComputeCommitment(ks, lg1) // Flush remaining nodes to nodes channel t.Flush(nodesCh) + comm := t.ComputeCommitment(ks, lg1) root := common.BytesToHash(bls.ToCompressedG1(comm)) out <- root } @@ -514,9 +514,9 @@ func computeCommitment(ctx *cli.Context) error { go func() { for fn := range nodesCh { nodesCount++ - value, err := rlp.EncodeToBytes(fn.Node) + value, err := fn.Node.Serialize() if err != nil { - log.Error("Failed to encode verkle node", "error", err) + log.Error("Failed to serialize verkle node", "error", err) } if err := verkledb.Put(fn.Hash[:], value); err != nil { log.Error("Failed to write verkle node to db", "error", err)