diff --git a/admin/gcp/main.go b/admin/gcp/main.go new file mode 100644 index 00000000..3846f68b --- /dev/null +++ b/admin/gcp/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "fmt" + "io" + "math" + + gcloud "cloud.google.com/go/storage" + "github.com/fxamacker/cbor/v2" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/engine/execution/ingestion/uploader" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow/protobuf/go/flow/access" + execData "github.com/onflow/flow/protobuf/go/flow/executiondata" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func CompareTrieUpdates(e []*ledger.TrieUpdate, g []*ledger.TrieUpdate) error { + // compare lengths + if len(e) != len(g) { + return fmt.Errorf("unequal lengths") + } + // compare updates + for idx, update := range e { + if !update.Equals(g[idx]) { + return fmt.Errorf("mismatching trie update at index %d", idx) + } + } + return nil +} + +func CompareCborRegistersWithExecutionResult( + ctx context.Context, + a access.AccessAPIClient, + e execData.ExecutionDataAPIClient, + b gcloud.BucketHandle, + height uint64, + decoder cbor.DecMode, + chain flow.Chain, +) error { + // get block and determine file name + req := access.GetBlockByHeightRequest{Height: height, FullBlockResponse: true} + res, err := a.GetBlockByHeight(ctx, &req) + if err != nil { + panic(fmt.Errorf("could not get block for height %d: %w", height, err)) + } + block, err := convert.MessageToBlock(res.Block) + if err != nil { + panic(fmt.Errorf("could not convert message to block: %w", err)) + } + fileName := block.ID().String() + ".cbor" + print(fileName) + // get cbor file and unmarhsall to block data + object := b.Object(fileName) + reader, err := object.NewReader(context.Background()) + if err != nil { + panic(fmt.Errorf("could not create object reader: %w", err)) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + panic(fmt.Errorf("could not read execution record: %w", err)) + } + + var record uploader.BlockData + err = decoder.Unmarshal(data, &record) + if err != nil { + panic(fmt.Errorf("could not decode execution record: %w", err)) + } + + // get execution result for the block to compare trieUpdates + req2 := execData.GetExecutionDataByBlockIDRequest{BlockId: convert.IdentifierToMessage(block.ID())} + res2, err := e.GetExecutionDataByBlockID(ctx, &req2) + if err != nil { + panic(fmt.Errorf("could not get execution data for height %d: %w", height, err)) + } + extractedData, err := convert.MessageToBlockExecutionData(res2.BlockExecutionData, chain) + // extract trie Updates + trieUpdates := make([]*ledger.TrieUpdate, 0, 0) + for _, chunk := range extractedData.ChunkExecutionDatas { + trieUpdates = append(trieUpdates, chunk.TrieUpdate) + } + // compare + return CompareTrieUpdates(trieUpdates, record.TrieUpdates) +} + +func main() { + accessAddr := "access-002.devnet46.nodes.onflow.org:9000" + gbucketName := "flow_public_devnet46_execution_state" + gClient, err := gcloud.NewClient(context.Background(), + option.WithoutAuthentication(), + ) + bucket := gClient.Bucket(gbucketName) + + decOptions := cbor.DecOptions{ + ExtraReturnErrors: cbor.ExtraDecErrorUnknownField, + MaxArrayElements: math.MaxInt64, + } + decoder, err := decOptions.DecMode() + if err != nil { + panic(fmt.Errorf("could not create decoder: %w", err)) + } + defer func() { + err := gClient.Close() + if err != nil { + + } + }() + // initialize clients + opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(40 * 1024 * 1024)), + grpc.WithTransportCredentials(insecure.NewCredentials())} + conn1, err := grpc.Dial(accessAddr, opts...) + if err != nil { + panic(err) + } + conn2, err := grpc.Dial(accessAddr, opts...) + if err != nil { + panic(err) + } + execDataAPI := execData.NewExecutionDataAPIClient(conn1) + accessAPI := access.NewAccessAPIClient(conn2) + // set chain + chain := flow.Testnet.Chain() + var start uint64 = 110950403 + var end uint64 = 110950404 + for height := start; height < end; height++ { + CompareCborRegistersWithExecutionResult(context.Background(), accessAPI, execDataAPI, *bucket, height, decoder, + chain) + } +} diff --git a/go.mod b/go.mod index d61d4e1b..c2b5ae16 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/klauspost/compress v1.15.15 github.com/onflow/cadence v0.39.14 - github.com/onflow/flow-go-sdk v0.41.5 + github.com/onflow/flow-go-sdk v0.41.6 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/tsdb v0.7.1 github.com/rs/zerolog v1.29.0 @@ -28,7 +28,7 @@ require ( require ( github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b github.com/envoyproxy/protoc-gen-validate v0.9.1 - github.com/onflow/flow-go v0.31.9 + github.com/onflow/flow-go v0.31.11 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 go.opentelemetry.io/otel v1.14.0 diff --git a/go.sum b/go.sum index 8ae7e766..b586bc03 100644 --- a/go.sum +++ b/go.sum @@ -1211,10 +1211,10 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+K github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go v0.31.9 h1:8wfSB8TGfs4PuuafG4xqkstDpsRfnYIzZ1d7o091aCA= -github.com/onflow/flow-go v0.31.9/go.mod h1:pHI5I/O3EWTWAZv5NCQbw3w3T8T3aB++ip8dueCTSnc= -github.com/onflow/flow-go-sdk v0.41.5 h1:/nBYCo8TtTcin/+SmDF2t8vl++NjKhpaCS24UHkfYfw= -github.com/onflow/flow-go-sdk v0.41.5/go.mod h1:3y/cI1Q4jJz1+RmeWkV36D+hAB5bk2SD1oqGvIudbJ4= +github.com/onflow/flow-go v0.31.11 h1:dgDfj2P+I3ZdMBlXyjaDFlxwtOL2cIChq0MDqGhBn+s= +github.com/onflow/flow-go v0.31.11/go.mod h1:kh6gggsQGIZYrXSIOskuKLAGJQxwpMXaIN3xFaljVQc= +github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= +github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4=