From 8899581ee6f590a8550fe87034e5f9d757daa9c8 Mon Sep 17 00:00:00 2001 From: Amlandeep Bhadra Date: Sun, 16 Jul 2023 15:10:36 -0400 Subject: [PATCH 1/2] initial commit --- admin/gcp/main.go | 97 +++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 8 ++-- 3 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 admin/gcp/main.go diff --git a/admin/gcp/main.go b/admin/gcp/main.go new file mode 100644 index 00000000..22fcde98 --- /dev/null +++ b/admin/gcp/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "fmt" + "io" + "math" + + gcloud "cloud.google.com/go/storage" + "github.com/fxamacker/cbor/v2" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/engine/execution/ingestion/uploader" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow/protobuf/go/flow/access" + "google.golang.org/api/option" +) + +func CompareTrieUpdates(a []*ledger.TrieUpdate, b []*ledger.TrieUpdate) error { + +} + +func CompareCborRegistersWithExecutionResult( + ctx context.Context, + a access.AccessAPIClient, + e access., + b *gcloud.BucketHandle, + height uint64, + decoder cbor.DecMode, +) error { + // get block and determine file name + req := access.GetBlockByHeightRequest{Height: height, FullBlockResponse: true} + res, err := a.GetBlockByHeight(ctx, &req) + if err != nil { + panic(fmt.Errorf("could not get block for height %d: %w", height, err)) + } + block, err := convert.MessageToBlock(res.Block) + if err != nil { + panic(fmt.Errorf("could not convert message to block: %w", err)) + } + fileName := block.ID().String() + ".cbor" + + // get cbor file and unmarhsall to block data + object := b.Object(fileName) + reader, err := object.NewReader(context.Background()) + if err != nil { + panic(fmt.Errorf("could not create object reader: %w", err)) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + panic(fmt.Errorf("could not read execution record: %w", err)) + } + + var record uploader.BlockData + err = decoder.Unmarshal(data, &record) + if err != nil { + panic(fmt.Errorf("could not decode execution record: %w", err)) + } + + // get execution result for the block to compare trieUpdates + req2 := access.GetExecutionResultForBlockIDRequest{BlockId: convert.IdentifierToMessage(block.ID())} + res2, err := a.GetExecutionResultForBlockID(ctx, &req2) + if err != nil { + panic(fmt.Errorf("could not get execution result for height %d: %w", height, err)) + } + executionResult, err := convert.MessageToExecutionResult(res2.ExecutionResult) + if err != nil { + + } + + err = CompareTrieUpdates() +} + +func main() { + gbucketName := "" + gClient, err := gcloud.NewClient(context.Background(), + option.WithoutAuthentication(), + ) + bucket := gClient.Bucket(gbucketName) + + decOptions := cbor.DecOptions{ + ExtraReturnErrors: cbor.ExtraDecErrorUnknownField, + MaxArrayElements: math.MaxInt64, + } + decoder, err := decOptions.DecMode() + if err != nil { + panic(fmt.Errorf("could not create decoder: %w", err)) + } + defer func() { + err := gClient.Close() + if err != nil { + + } + }() + CompareCborRegistersWithExecutionResult(context.Background()) +} diff --git a/go.mod b/go.mod index d61d4e1b..c2b5ae16 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/klauspost/compress v1.15.15 github.com/onflow/cadence v0.39.14 - github.com/onflow/flow-go-sdk v0.41.5 + github.com/onflow/flow-go-sdk v0.41.6 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/tsdb v0.7.1 github.com/rs/zerolog v1.29.0 @@ -28,7 +28,7 @@ require ( require ( github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b github.com/envoyproxy/protoc-gen-validate v0.9.1 - github.com/onflow/flow-go v0.31.9 + github.com/onflow/flow-go v0.31.11 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 go.opentelemetry.io/otel v1.14.0 diff --git a/go.sum b/go.sum index 8ae7e766..b586bc03 100644 --- a/go.sum +++ b/go.sum @@ -1211,10 +1211,10 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+K github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go v0.31.9 h1:8wfSB8TGfs4PuuafG4xqkstDpsRfnYIzZ1d7o091aCA= -github.com/onflow/flow-go v0.31.9/go.mod h1:pHI5I/O3EWTWAZv5NCQbw3w3T8T3aB++ip8dueCTSnc= -github.com/onflow/flow-go-sdk v0.41.5 h1:/nBYCo8TtTcin/+SmDF2t8vl++NjKhpaCS24UHkfYfw= -github.com/onflow/flow-go-sdk v0.41.5/go.mod h1:3y/cI1Q4jJz1+RmeWkV36D+hAB5bk2SD1oqGvIudbJ4= +github.com/onflow/flow-go v0.31.11 h1:dgDfj2P+I3ZdMBlXyjaDFlxwtOL2cIChq0MDqGhBn+s= +github.com/onflow/flow-go v0.31.11/go.mod h1:kh6gggsQGIZYrXSIOskuKLAGJQxwpMXaIN3xFaljVQc= +github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= +github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= From 67dd44bc05504cdff3aa1c3b9f81180db50b9967 Mon Sep 17 00:00:00 2001 From: Amlandeep Bhadra Date: Sun, 16 Jul 2023 16:25:06 -0400 Subject: [PATCH 2/2] implement exec data comparison --- admin/gcp/main.go | 68 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/admin/gcp/main.go b/admin/gcp/main.go index 22fcde98..3846f68b 100644 --- a/admin/gcp/main.go +++ b/admin/gcp/main.go @@ -11,21 +11,36 @@ import ( "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/engine/execution/ingestion/uploader" "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow/protobuf/go/flow/access" + execData "github.com/onflow/flow/protobuf/go/flow/executiondata" "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) -func CompareTrieUpdates(a []*ledger.TrieUpdate, b []*ledger.TrieUpdate) error { - +func CompareTrieUpdates(e []*ledger.TrieUpdate, g []*ledger.TrieUpdate) error { + // compare lengths + if len(e) != len(g) { + return fmt.Errorf("unequal lengths") + } + // compare updates + for idx, update := range e { + if !update.Equals(g[idx]) { + return fmt.Errorf("mismatching trie update at index %d", idx) + } + } + return nil } func CompareCborRegistersWithExecutionResult( ctx context.Context, a access.AccessAPIClient, - e access., - b *gcloud.BucketHandle, + e execData.ExecutionDataAPIClient, + b gcloud.BucketHandle, height uint64, decoder cbor.DecMode, + chain flow.Chain, ) error { // get block and determine file name req := access.GetBlockByHeightRequest{Height: height, FullBlockResponse: true} @@ -38,7 +53,7 @@ func CompareCborRegistersWithExecutionResult( panic(fmt.Errorf("could not convert message to block: %w", err)) } fileName := block.ID().String() + ".cbor" - + print(fileName) // get cbor file and unmarhsall to block data object := b.Object(fileName) reader, err := object.NewReader(context.Background()) @@ -59,21 +74,24 @@ func CompareCborRegistersWithExecutionResult( } // get execution result for the block to compare trieUpdates - req2 := access.GetExecutionResultForBlockIDRequest{BlockId: convert.IdentifierToMessage(block.ID())} - res2, err := a.GetExecutionResultForBlockID(ctx, &req2) + req2 := execData.GetExecutionDataByBlockIDRequest{BlockId: convert.IdentifierToMessage(block.ID())} + res2, err := e.GetExecutionDataByBlockID(ctx, &req2) if err != nil { - panic(fmt.Errorf("could not get execution result for height %d: %w", height, err)) + panic(fmt.Errorf("could not get execution data for height %d: %w", height, err)) } - executionResult, err := convert.MessageToExecutionResult(res2.ExecutionResult) - if err != nil { - + extractedData, err := convert.MessageToBlockExecutionData(res2.BlockExecutionData, chain) + // extract trie Updates + trieUpdates := make([]*ledger.TrieUpdate, 0, 0) + for _, chunk := range extractedData.ChunkExecutionDatas { + trieUpdates = append(trieUpdates, chunk.TrieUpdate) } - - err = CompareTrieUpdates() + // compare + return CompareTrieUpdates(trieUpdates, record.TrieUpdates) } func main() { - gbucketName := "" + accessAddr := "access-002.devnet46.nodes.onflow.org:9000" + gbucketName := "flow_public_devnet46_execution_state" gClient, err := gcloud.NewClient(context.Background(), option.WithoutAuthentication(), ) @@ -93,5 +111,25 @@ func main() { } }() - CompareCborRegistersWithExecutionResult(context.Background()) + // initialize clients + opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(40 * 1024 * 1024)), + grpc.WithTransportCredentials(insecure.NewCredentials())} + conn1, err := grpc.Dial(accessAddr, opts...) + if err != nil { + panic(err) + } + conn2, err := grpc.Dial(accessAddr, opts...) + if err != nil { + panic(err) + } + execDataAPI := execData.NewExecutionDataAPIClient(conn1) + accessAPI := access.NewAccessAPIClient(conn2) + // set chain + chain := flow.Testnet.Chain() + var start uint64 = 110950403 + var end uint64 = 110950404 + for height := start; height < end; height++ { + CompareCborRegistersWithExecutionResult(context.Background(), accessAPI, execDataAPI, *bucket, height, decoder, + chain) + } }