New feature 25 "Deterministic Finality and Ride V9" added.

.github/workflows/security.yml

@@ -35,15 +35,15 @@ jobs:
         uses: securego/gosec@424fc4cd9c82ea0fd6bee9cd49c2db2c3cc0c93f # v2.22.11
         with:
           # with '-no-fail' we let the report trigger content trigger a failure using the GitHub Security features.
-          args: "-no-fail -fmt sarif -out gosec.sarif ./..."
+          args: "-no-fail -exclude-generated -fmt sarif -out gosec.sarif ./..."
       - name: Transform gosec SARIF to meet SARIF 2.1.0 schema requirements
         # Produces SARIF file has two incompatibilities with SARIF 2.1.0 schema all of them are in `fixes` object:
         #  1. Field `description` contains only `markdown` property, but `text` property is mandatory.
         #  2. Property `artifactChanges` must be non-empty array with valid `artifactChange` object.
         # To fix those issues the first part of jq query copies content of `markdown` field into `text` field
         # if it's not present. In the second part if the `fixes` object has null array as `artifactChanges` the empty
         # valid `artifactChange` object inserted.
-        # After fix of mentioned gosec issues this step can be removed. 
+        # After fix of mentioned gosec issues this step can be removed.
         run: |
           set -e
           jq 'walk(if type=="object" and has("markdown") and (.text == null or .text == "") then .text = .markdown else . end) | (.runs[].results[]? .fixes[]? .artifactChanges) |= (if . == null then [{"artifactLocation": { "uri": "" }, "replacements": [{"deletedRegion": {"byteOffset": 0, "byteLength": 0 }}]}] else . end)' gosec.sarif > gosec_fixed.sarif

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "pkg/grpc/protobuf-schemas"]
 	path = pkg/grpc/protobuf-schemas
 	url = https://github.com/wavesplatform/protobuf-schemas
+	branch = deterministic-finality

.semgrepignore

@@ -18,3 +18,6 @@ cmd/wmd/internal/swagger/
 
 # Ignore testdata folders
 pkg/state/testdata/
+
+# Ignore generated files
+**/*.pb.go

cmd/statecmp/statecmp.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/wavesplatform/gowaves/pkg/client"
+	"github.com/wavesplatform/gowaves/pkg/crypto"
 	"github.com/wavesplatform/gowaves/pkg/logging"
 	"github.com/wavesplatform/gowaves/pkg/proto"
 )
@@ -43,10 +44,10 @@ func checkAndUpdateURL(s string) (string, error) {
 	return u.String(), nil
 }
 
-func loadStateHash(ctx context.Context, cl *client.Client, height uint64, tries int) (*proto.StateHash, error) {
+func loadStateHash(ctx context.Context, cl *client.Client, height uint64, tries int) (proto.StateHash, error) {
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
-	var sh *proto.StateHash
+	var sh proto.StateHash
 	var err error
 	for range tries {
 		sh, _, err = cl.Debug.StateHash(ctx, height)
@@ -61,12 +62,12 @@ type printer struct {
 	lock sync.Mutex
 }
 
-func (p *printer) printDifferentResults(height uint64, res map[proto.FieldsHashes]*nodesGroup) {
+func (p *printer) printDifferentResults(height uint64, res map[crypto.Digest]*nodesGroup) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
-	for fh, nodes := range res {
-		hashJs, err := json.Marshal(fh)
+	for sh, nodes := range res {
+		hashJs, err := json.Marshal(sh)
 		if err != nil {
 			panic(err)
 		}
@@ -76,7 +77,7 @@ func (p *printer) printDifferentResults(height uint64, res map[proto.FieldsHashe
 }
 
 type stateHashInfo struct {
-	hash *proto.StateHash
+	sh   proto.StateHash
 	node string
 }
 
@@ -87,12 +88,13 @@ type hashResult struct {
 
 type nodesGroup struct {
 	nodes []string
+	sh    proto.StateHash
 }
 
-func newNodesGroup(first string) *nodesGroup {
+func newNodesGroup(first string, sh proto.StateHash) *nodesGroup {
 	nodes := make([]string, 1)
 	nodes[0] = first
-	return &nodesGroup{nodes: nodes}
+	return &nodesGroup{nodes: nodes, sh: sh}
 }
 
 func (ng *nodesGroup) addNode(node string) {
@@ -112,27 +114,27 @@ func manageHeight(
 			sh, err := loadStateHash(ctx, cl, height, tries)
 			res := hashResult{
 				res: stateHashInfo{
-					hash: sh,
+					sh:   sh,
 					node: node,
 				},
 				err: err,
 			}
 			results <- res
 		}(cl, node)
 	}
-	differentResults := make(map[proto.FieldsHashes]*nodesGroup)
+	differentResults := make(map[crypto.Digest]*nodesGroup)
 	for range clients {
 		hr := <-results
 		if hr.err != nil {
 			cancel()
 			return hr.err
 		}
-		fh := hr.res.hash.FieldsHashes
-		nodesGroup, ok := differentResults[fh]
+		sh := hr.res.sh
+		ng, ok := differentResults[sh.GetSumHash()]
 		if !ok {
-			differentResults[fh] = newNodesGroup(hr.res.node)
+			differentResults[sh.GetSumHash()] = newNodesGroup(hr.res.node, sh)
 		} else {
-			nodesGroup.addNode(hr.res.node)
+			ng.addNode(hr.res.node)
 		}
 	}
 	if len(differentResults) != 1 {

cmd/statehash/statehash.go

@@ -252,7 +252,7 @@ func findLastEqualStateHashes(
 ) (uint64, error) {
 	var err error
 	var r uint64
-	var lsh, rsh *proto.StateHashDebug
+	var lsh, rsh proto.StateHashDebug
 	var start uint64 = 1
 	for start <= stop {
 		middle := (start + stop) / 2
@@ -279,7 +279,7 @@ func findLastEqualStateHashes(
 	return r, nil
 }
 
-func stateHashToString(sh *proto.StateHashDebug) string {
+func stateHashToString(sh proto.StateHashDebug) string {
 	js, err := json.Marshal(sh)
 	if err != nil {
 		slog.Error("Failed to render state hash to text", logging.Error(err))
@@ -288,24 +288,25 @@ func stateHashToString(sh *proto.StateHashDebug) string {
 	return string(js)
 }
 
-func compareStateHashes(sh1, sh2 *proto.StateHashDebug, onlyLegacy bool) (bool, error) {
-	if sh1.BlockID != sh2.BlockID {
-		return false, fmt.Errorf("different block IDs: '%s' != '%s'", sh1.BlockID.String(), sh2.BlockID.String())
+func compareStateHashes(sh1, sh2 proto.StateHashDebug, onlyLegacy bool) (bool, error) {
+	if sh1.GetBlockID() != sh2.GetBlockID() {
+		return false, fmt.Errorf("different block IDs: '%s' != '%s'",
+			sh1.GetBlockID().String(), sh2.GetBlockID().String())
 	}
-	legacyEqual := sh1.SumHash == sh2.SumHash
+	legacyEqual := sh1.GetSumHash() == sh2.GetSumHash()
 	if onlyLegacy {
 		return legacyEqual, nil
 	}
-	return legacyEqual && sh1.SnapshotHash == sh2.SnapshotHash, nil
+	return legacyEqual && sh1.GetSnapshotHash() == sh2.GetSnapshotHash(), nil
 }
 
 func compareWithRemote(
 	ctx context.Context,
-	sh *proto.StateHashDebug,
+	sh proto.StateHashDebug,
 	c *client.Client,
 	h uint64,
 	onlyLegacy bool,
-) (bool, *proto.StateHashDebug, error) {
+) (bool, proto.StateHashDebug, error) {
 	rsh, err := getRemoteStateHash(ctx, c, h)
 	if err != nil {
 		return false, nil, err
@@ -314,15 +315,15 @@ func compareWithRemote(
 	return ok, rsh, err
 }
 
-func getRemoteStateHash(ctx context.Context, c *client.Client, h uint64) (*proto.StateHashDebug, error) {
+func getRemoteStateHash(ctx context.Context, c *client.Client, h uint64) (proto.StateHashDebug, error) {
 	sh, _, err := c.Debug.StateHashDebug(ctx, h)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get state hash at %d height: %w", h, err)
 	}
 	return sh, nil
 }
 
-func getLocalStateHash(st state.StateInfo, h uint64) (*proto.StateHashDebug, error) {
+func getLocalStateHash(st state.StateInfo, h uint64) (proto.StateHashDebug, error) {
 	const localVersion = "local"
 	lsh, err := st.LegacyStateHashAtHeight(h)
 	if err != nil {
@@ -332,8 +333,15 @@ func getLocalStateHash(st state.StateInfo, h uint64) (*proto.StateHashDebug, err
 	if err != nil {
 		return nil, fmt.Errorf("failed to get snapshot state hash at %d height: %w", h, err)
 	}
-	shd := proto.NewStateHashJSDebug(*lsh, h, localVersion, snapSH)
-	return &shd, nil
+	finalityActivated, err := st.IsActiveAtHeight(int16(settings.DeterministicFinality), h)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine finality activation at %d height: %w", h, err)
+	}
+	bh, bhErr := st.BlockByHeight(h)
+	if bhErr != nil {
+		return nil, fmt.Errorf("failed to get block at %d height: %w", h, bhErr)
+	}
+	return proto.NewStateHashDebug(finalityActivated, lsh, h, localVersion, snapSH, bh.BaseTarget)
 }
 
 func showUsage() {

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/ccoveille/go-safecast/v2 v2.0.0
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/cloudflare/circl v1.6.1
 	github.com/consensys/gnark v0.14.0
 	github.com/consensys/gnark-crypto v0.19.2
 	github.com/coocood/freecache v1.2.4

go.sum

@@ -37,6 +37,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
+github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/consensys/gnark v0.14.0 h1:RG+8WxRanFSFBSlmCDRJnYMYYKpH3Ncs5SMzg24B5HQ=
 github.com/consensys/gnark v0.14.0/go.mod h1:1IBpDPB/Rdyh55bQRR4b0z1WvfHQN1e0020jCvKP2Gk=

itests/clients/http_client.go

@@ -21,7 +21,7 @@ type HTTPClient struct {
 	timeout time.Duration
 }
 
-func NewHTTPClient(t *testing.T, impl Implementation, port string) *HTTPClient {
+func NewHTTPClient(t testing.TB, impl Implementation, port string) *HTTPClient {
 	c, err := client.NewClient(client.Options{
 		BaseUrl: "http://" + config.DefaultIP + ":" + port + "/",
 		Client:  &http.Client{Timeout: d.DefaultTimeout},
@@ -37,7 +37,7 @@ func NewHTTPClient(t *testing.T, impl Implementation, port string) *HTTPClient {
 	}
 }
 
-func (c *HTTPClient) GetHeight(t *testing.T, opts ...config.WaitOption) *client.BlocksHeight {
+func (c *HTTPClient) GetHeight(t testing.TB, opts ...config.WaitOption) *client.BlocksHeight {
 	params := config.NewWaitParams(opts...)
 	ctx, cancel := context.WithTimeout(params.Ctx, params.Timeout)
 	defer cancel()
@@ -46,7 +46,7 @@ func (c *HTTPClient) GetHeight(t *testing.T, opts ...config.WaitOption) *client.
 	return h
 }
 
-func (c *HTTPClient) StateHash(t *testing.T, height uint64, opts ...config.WaitOption) *proto.StateHash {
+func (c *HTTPClient) StateHash(t testing.TB, height uint64, opts ...config.WaitOption) proto.StateHash {
 	params := config.NewWaitParams(opts...)
 	ctx, cancel := context.WithTimeout(params.Ctx, params.Timeout)
 	defer cancel()
@@ -55,7 +55,7 @@ func (c *HTTPClient) StateHash(t *testing.T, height uint64, opts ...config.WaitO
 	return stateHash
 }
 
-func (c *HTTPClient) PrintMsg(t *testing.T, msg string) {
+func (c *HTTPClient) PrintMsg(t testing.TB, msg string) {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 	_, err := c.cli.Debug.PrintMsg(ctx, msg)
@@ -69,7 +69,7 @@ func (c *HTTPClient) GetAssetDetails(assetID crypto.Digest) (*client.AssetsDetai
 	return details, err
 }
 
-func (c *HTTPClient) TransactionInfo(t *testing.T, id crypto.Digest) proto.Transaction {
+func (c *HTTPClient) TransactionInfo(t testing.TB, id crypto.Digest) proto.Transaction {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 	info, _, err := c.cli.Transactions.Info(ctx, id)
@@ -89,7 +89,7 @@ func (c *HTTPClient) TransactionBroadcast(transaction proto.Transaction) (*clien
 	return c.cli.Transactions.Broadcast(ctx, transaction)
 }
 
-func (c *HTTPClient) WavesBalance(t *testing.T, address proto.WavesAddress) *client.AddressesBalance {
+func (c *HTTPClient) WavesBalance(t testing.TB, address proto.WavesAddress) *client.AddressesBalance {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 	balance, _, err := c.cli.Addresses.Balance(ctx, address)
@@ -98,7 +98,7 @@ func (c *HTTPClient) WavesBalance(t *testing.T, address proto.WavesAddress) *cli
 }
 
 func (c *HTTPClient) AssetBalance(
-	t *testing.T, address proto.WavesAddress, assetID crypto.Digest,
+	t testing.TB, address proto.WavesAddress, assetID crypto.Digest,
 ) *client.AssetsBalanceAndAsset {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
@@ -118,7 +118,7 @@ func (c *HTTPClient) ConnectedPeersCtx(ctx context.Context) ([]*client.PeersConn
 	return connectedPeers, resp, err
 }
 
-func (c *HTTPClient) BlockHeader(t *testing.T, height proto.Height) *client.Headers {
+func (c *HTTPClient) BlockHeader(t testing.TB, height proto.Height) *client.Headers {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 
@@ -127,7 +127,7 @@ func (c *HTTPClient) BlockHeader(t *testing.T, height proto.Height) *client.Head
 	return header
 }
 
-func (c *HTTPClient) Rewards(t *testing.T) *client.RewardInfo {
+func (c *HTTPClient) Rewards(t testing.TB) *client.RewardInfo {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 
@@ -136,7 +136,7 @@ func (c *HTTPClient) Rewards(t *testing.T) *client.RewardInfo {
 	return rewardInfo
 }
 
-func (c *HTTPClient) RewardsAtHeight(t *testing.T, height proto.Height) *client.RewardInfo {
+func (c *HTTPClient) RewardsAtHeight(t testing.TB, height proto.Height) *client.RewardInfo {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()
 
@@ -145,7 +145,7 @@ func (c *HTTPClient) RewardsAtHeight(t *testing.T, height proto.Height) *client.
 	return rewardInfo
 }
 
-func (c *HTTPClient) RollbackToHeight(t *testing.T, height uint64, returnTxToUtx bool) *proto.BlockID {
+func (c *HTTPClient) RollbackToHeight(t testing.TB, height uint64, returnTxToUtx bool) *proto.BlockID {
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
 	defer cancel()