Amlandeep/create register validator

admin/gcp/main.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+
+	gcloud "cloud.google.com/go/storage"
+	"github.com/fxamacker/cbor/v2"
+	"github.com/onflow/flow-go/engine/common/rpc/convert"
+	"github.com/onflow/flow-go/engine/execution/ingestion/uploader"
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow/protobuf/go/flow/access"
+	execData "github.com/onflow/flow/protobuf/go/flow/executiondata"
+	"google.golang.org/api/option"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func CompareTrieUpdates(e []*ledger.TrieUpdate, g []*ledger.TrieUpdate) error {
+	// compare lengths
+	if len(e) != len(g) {
+		return fmt.Errorf("unequal lengths")
+	}
+	// compare updates
+	for idx, update := range e {
+		if !update.Equals(g[idx]) {
+			return fmt.Errorf("mismatching trie update at index %d", idx)
+		}
+	}
+	return nil
+}
+
+func CompareCborRegistersWithExecutionResult(
+	ctx context.Context,
+	a access.AccessAPIClient,
+	e execData.ExecutionDataAPIClient,
+	b gcloud.BucketHandle,
+	height uint64,
+	decoder cbor.DecMode,
+	chain flow.Chain,
+) error {
+	// get block and determine file name
+	req := access.GetBlockByHeightRequest{Height: height, FullBlockResponse: true}
+	res, err := a.GetBlockByHeight(ctx, &req)
+	if err != nil {
+		panic(fmt.Errorf("could not get block for height %d: %w", height, err))
+	}
+	block, err := convert.MessageToBlock(res.Block)
+	if err != nil {
+		panic(fmt.Errorf("could not convert message to block: %w", err))
+	}
+	fileName := block.ID().String() + ".cbor"
+	print(fileName)
+	// get cbor file and unmarhsall to block data
+	object := b.Object(fileName)
+	reader, err := object.NewReader(context.Background())
+	if err != nil {
+		panic(fmt.Errorf("could not create object reader: %w", err))
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		panic(fmt.Errorf("could not read execution record: %w", err))
+	}
+
+	var record uploader.BlockData
+	err = decoder.Unmarshal(data, &record)
+	if err != nil {
+		panic(fmt.Errorf("could not decode execution record: %w", err))
+	}
+
+	// get execution result for the block to compare trieUpdates
+	req2 := execData.GetExecutionDataByBlockIDRequest{BlockId: convert.IdentifierToMessage(block.ID())}
+	res2, err := e.GetExecutionDataByBlockID(ctx, &req2)
+	if err != nil {
+		panic(fmt.Errorf("could not get execution data for height %d: %w", height, err))
+	}
+	extractedData, err := convert.MessageToBlockExecutionData(res2.BlockExecutionData, chain)
+	// extract trie Updates
+	trieUpdates := make([]*ledger.TrieUpdate, 0, 0)
+	for _, chunk := range extractedData.ChunkExecutionDatas {
+		trieUpdates = append(trieUpdates, chunk.TrieUpdate)
+	}
+	// compare
+	return CompareTrieUpdates(trieUpdates, record.TrieUpdates)
+}
+
+func main() {
+	accessAddr := "access-002.devnet46.nodes.onflow.org:9000"
+	gbucketName := "flow_public_devnet46_execution_state"
+	gClient, err := gcloud.NewClient(context.Background(),
+		option.WithoutAuthentication(),
+	)
+	bucket := gClient.Bucket(gbucketName)
+
+	decOptions := cbor.DecOptions{
+		ExtraReturnErrors: cbor.ExtraDecErrorUnknownField,
+		MaxArrayElements:  math.MaxInt64,
+	}
+	decoder, err := decOptions.DecMode()
+	if err != nil {
+		panic(fmt.Errorf("could not create decoder: %w", err))
+	}
+	defer func() {
+		err := gClient.Close()
+		if err != nil {
+
+		}
+	}()
+	// initialize clients
+	opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(40 * 1024 * 1024)),
+		grpc.WithTransportCredentials(insecure.NewCredentials())}
+	conn1, err := grpc.Dial(accessAddr, opts...)
+	if err != nil {
+		panic(err)
+	}
+	conn2, err := grpc.Dial(accessAddr, opts...)
+	if err != nil {
+		panic(err)
+	}
+	execDataAPI := execData.NewExecutionDataAPIClient(conn1)
+	accessAPI := access.NewAccessAPIClient(conn2)
+	// set chain
+	chain := flow.Testnet.Chain()
+	var start uint64 = 110950403
+	var end uint64 = 110950404
+	for height := start; height < end; height++ {
+		CompareCborRegistersWithExecutionResult(context.Background(), accessAPI, execDataAPI, *bucket, height, decoder,
+			chain)
+	}
+}

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/klauspost/compress v1.15.15
 	github.com/onflow/cadence v0.39.14
-	github.com/onflow/flow-go-sdk v0.41.5
+	github.com/onflow/flow-go-sdk v0.41.6
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/tsdb v0.7.1
 	github.com/rs/zerolog v1.29.0
@@ -28,7 +28,7 @@ require (
 require (
 	github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b
 	github.com/envoyproxy/protoc-gen-validate v0.9.1
-	github.com/onflow/flow-go v0.31.9
+	github.com/onflow/flow-go v0.31.11
 	github.com/onflow/flow-go/crypto v0.24.7
 	github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391
 	go.opentelemetry.io/otel v1.14.0

go.sum

@@ -1211,10 +1211,10 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+K
 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A=
 github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE=
 github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU=
-github.com/onflow/flow-go v0.31.9 h1:8wfSB8TGfs4PuuafG4xqkstDpsRfnYIzZ1d7o091aCA=
-github.com/onflow/flow-go v0.31.9/go.mod h1:pHI5I/O3EWTWAZv5NCQbw3w3T8T3aB++ip8dueCTSnc=
-github.com/onflow/flow-go-sdk v0.41.5 h1:/nBYCo8TtTcin/+SmDF2t8vl++NjKhpaCS24UHkfYfw=
-github.com/onflow/flow-go-sdk v0.41.5/go.mod h1:3y/cI1Q4jJz1+RmeWkV36D+hAB5bk2SD1oqGvIudbJ4=
+github.com/onflow/flow-go v0.31.11 h1:dgDfj2P+I3ZdMBlXyjaDFlxwtOL2cIChq0MDqGhBn+s=
+github.com/onflow/flow-go v0.31.11/go.mod h1:kh6gggsQGIZYrXSIOskuKLAGJQxwpMXaIN3xFaljVQc=
+github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg=
+github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik=
 github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0=
 github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0=
 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4=