diff --git a/internal/migrations/001-common-actions.sql b/internal/migrations/001-common-actions.sql index 03c0f3285..6c243d3ac 100644 --- a/internal/migrations/001-common-actions.sql +++ b/internal/migrations/001-common-actions.sql @@ -1319,4 +1319,4 @@ CREATE OR REPLACE ACTION list_streams( CASE WHEN $order_by = 'stream_type ASC' THEN stream_type END ASC, CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC LIMIT $limit OFFSET $offset; -}; \ No newline at end of file +}; diff --git a/internal/migrations/021-metadata-actions.sql b/internal/migrations/021-metadata-actions.sql new file mode 100644 index 000000000..7480649c8 --- /dev/null +++ b/internal/migrations/021-metadata-actions.sql @@ -0,0 +1,80 @@ +/** + * list_metadata_by_height: Queries metadata within a specific block height range. + * Supports pagination. + * Supports filtering by key and ref. + * + * Parameters: + * $key: filter by metadata key. + *. $ref: filter by value reference. + * $from_height: Start height (inclusive). If NULL, uses earliest available. + * $to_height: End height (inclusive). If NULL, uses current height. + * $limit: Maximum number of results to return. + * $offset: Number of results to skip for pagination. + * + * Returns: + * Table with metadata entries matching the criteria. + */ +CREATE OR REPLACE ACTION list_metadata_by_height( + $key TEXT, + $ref TEXT, + $from_height INT8, + $to_height INT8, + $limit INT, + $offset INT +) PUBLIC view returns table( + stream_ref INT, + row_id uuid, + value_i INT, + value_f NUMERIC(36,18), + value_b bool, + value_s TEXT, + value_ref TEXT, + created_at INT8 +) { + -- Set defaults for pagination and validate values + if $limit IS NULL { + $limit := 1000; + } + if $offset IS NULL { + $offset := 0; + } + + -- Ensure non-negative values for PostgreSQL compatibility + if $limit < 0 { + $limit := 0; + } + if $offset < 0 { + $offset := 0; + } + + -- Get current block height for default behavior + $current_block INT8 := @height; + + -- Determine effective height range, if none given, get all metadata + $effective_from INT8 := COALESCE($from_height, 0); + $effective_to INT8 := COALESCE($to_height, $current_block); + + -- Validate height range + if $effective_from > $effective_to { + ERROR('Invalid height range: from_height (' || $effective_from::TEXT || ') > to_height (' || $effective_to::TEXT || ')'); + } + + RETURN SELECT + stream_ref, + row_id, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at + FROM metadata + WHERE metadata_key = $key + AND disabled_at IS NULL + -- do not use LOWER on value_ref, or it will break the index lookup + AND ($ref IS NULL OR value_ref = LOWER($ref)) + AND created_at >= $effective_from + AND created_at <= $effective_to + ORDER BY created_at ASC + LIMIT $limit OFFSET $offset; +}; \ No newline at end of file diff --git a/tests/streams/query/metadata_test.go b/tests/streams/query/metadata_test.go index d02ea7715..22819cadb 100644 --- a/tests/streams/query/metadata_test.go +++ b/tests/streams/query/metadata_test.go @@ -3,6 +3,7 @@ package tests import ( "context" "fmt" + "strings" "testing" "github.com/pkg/errors" @@ -14,6 +15,7 @@ import ( testutils "github.com/trufnetwork/node/tests/streams/utils" "github.com/trufnetwork/node/tests/streams/utils/procedure" "github.com/trufnetwork/node/tests/streams/utils/setup" + "github.com/trufnetwork/node/tests/streams/utils/table" "github.com/trufnetwork/sdk-go/core/types" "github.com/trufnetwork/sdk-go/core/util" ) @@ -29,14 +31,19 @@ var ( } ) -// TestQUERY04MetadataInsertionAndRetrieval tests the insertion and retrieval of metadata +// TestQUERY04Metadata tests the insertion and retrieval of metadata // for both primitive and composed streams. Also tests disabling metadata and attempting to retrieve it. -func TestQUERY04MetadataInsertionAndRetrieval(t *testing.T) { +func TestQUERY04Metadata(t *testing.T) { kwilTesting.RunSchemaTest(t, kwilTesting.SchemaTest{ Name: "metadata_insertion_and_retrieval", FunctionTests: []kwilTesting.TestFunc{ WithMetadataTestSetup(testMetadataInsertionAndRetrieval(t, primitiveContractInfo)), WithMetadataTestSetup(testMetadataInsertionThenDisableAndRetrieval(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeight(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightNoKey(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightPagination(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightInvalidRange(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightInvalidPagination(t, primitiveContractInfo)), }, SeedScripts: migrations.GetSeedScriptPaths(), }, testutils.GetTestOptions()) @@ -161,3 +168,268 @@ func testMetadataInsertionThenDisableAndRetrieval(t *testing.T, contractInfo set return nil } } + +func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + expected := ` + | stream_ref | value_i | value_f | value_b | value_s | value_ref | created_at | + |---------------|-----------|---------------------|-----------------|--------|------------|----------------|------------|------------|------------| + | 1 | 0 | | | | | 1 | + | 1 | 1 | | | | | 1 |` + + table.AssertResultRowsEqualMarkdownTable(t, table.AssertResultRowsEqualMarkdownTableInput{ + Actual: result, + Expected: expected, + ExcludedColumns: []int{1}, + }) + + return nil + } +} + +func testListMetadataByHeightNoKey(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if (len(result) > 0) { // should return no rows + return errors.Wrapf(err, "expected empty results") + } + + return nil + } +} + +func testListMetadataByHeightPagination(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + limit := 2 + offset := 0 + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Offset: &offset, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if len(result) != 2 { + return errors.Errorf("expected 2 results, got %d", len(result)) + } + + offset = 2 + result, err = procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Offset: &offset, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if len(result) != 1 { + return errors.Errorf("expected 1 result, got %d", len(result)) + } + + return nil + } +} + +func testListMetadataByHeightInvalidRange(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + fromHeight := int64(10) + toHeight := int64(5) + _, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + FromHeight: &fromHeight, + ToHeight: &toHeight, + Height: 1, + }) + + if err == nil { + return errors.New("expected error for invalid height range (from_height > to_height), but got none") + } + + expectedError := "Invalid height range" + if !strings.Contains(err.Error(), expectedError) { + return errors.Errorf("expected error message to contain '%s', got: %s", expectedError, err.Error()) + } + + return nil + } +} + +func testListMetadataByHeightInvalidPagination(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + // negative limit + limit := -10 + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Height: 1, + }) + if err != nil { + return errors.Wrap(err, "unexpected error with negative limit") + } + + if len(result) != 0 { + return errors.Errorf("expected empty results with negative limit (converted to 0), got %d results", len(result)) + } + + negativeOffset := -5 + result, err = procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Offset: &negativeOffset, + Height: 1, + }) + if err != nil { + return errors.Wrap(err, "unexpected error with negative offset") + } + + return nil + } +} diff --git a/tests/streams/utils/procedure/execute.go b/tests/streams/utils/procedure/execute.go index b39994569..455778c43 100644 --- a/tests/streams/utils/procedure/execute.go +++ b/tests/streams/utils/procedure/execute.go @@ -843,3 +843,46 @@ func GetTaxonomiesForStreams(ctx context.Context, input GetTaxonomiesForStreamsI return processResultRows(resultRows) } + +// ListMetadataByHeight executes list_metadata_by_height action +func ListMetadataByHeight(ctx context.Context, input ListMetadataByHeightInput) ([]ResultRow, error) { + deployer, err := util.NewEthereumAddressFromBytes(input.Platform.Deployer) + if err != nil { + return nil, errors.Wrap(err, "error in ListMetadataByHeight.NewEthereumAddressFromBytes") + } + + txContext := &common.TxContext{ + Ctx: ctx, + BlockContext: &common.BlockContext{Height: input.Height}, + Signer: input.Platform.Deployer, + Caller: deployer.Address(), + TxID: input.Platform.Txid(), + } + + engineContext := &common.EngineContext{ + TxContext: txContext, + } + + var resultRows [][]any + r, err := input.Platform.Engine.Call(engineContext, input.Platform.DB, "", "list_metadata_by_height", []any{ + input.Key, + input.Value, + input.FromHeight, + input.ToHeight, + input.Limit, + input.Offset, + }, func(row *common.Row) error { + values := make([]any, len(row.Values)) + copy(values, row.Values) + resultRows = append(resultRows, values) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "error in ListMetadataByHeight.Call") + } + if r.Error != nil { + return nil, errors.Wrap(r.Error, "error in ListMetadataByHeight.Call") + } + + return processResultRows(resultRows) +} diff --git a/tests/streams/utils/procedure/types.go b/tests/streams/utils/procedure/types.go index 5b7b02cc5..37871636f 100644 --- a/tests/streams/utils/procedure/types.go +++ b/tests/streams/utils/procedure/types.go @@ -159,3 +159,14 @@ type GetTaxonomiesForStreamsInput struct { LatestOnly *bool Height int64 } + +type ListMetadataByHeightInput struct { + Platform *kwilTesting.Platform + Key string + Value *string + FromHeight *int64 + ToHeight *int64 + Limit *int + Offset *int + Height int64 +} diff --git a/tests/streams/utils/table/assert.go b/tests/streams/utils/table/assert.go index 17ba5e00b..6cf892573 100644 --- a/tests/streams/utils/table/assert.go +++ b/tests/streams/utils/table/assert.go @@ -13,6 +13,7 @@ type AssertResultRowsEqualMarkdownTableInput struct { Expected string ColumnTransformers map[string]func(string) string SortColumns []string + ExcludedColumns []int // drop only from actual } func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqualMarkdownTableInput) { @@ -21,16 +22,13 @@ func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqua t.Fatalf("error parsing expected markdown table: %v", err) } - // clear empty rows, because we won't get those from answer, but - // tests might include it just to be explicit about what is being tested + // Transform expected rows (apply transformers, keep all columns) expected := [][]string{} for _, row := range expectedTable.Rows { if row[1] != "" { transformedRow := make([]string, len(row)) for colIdx, value := range row { - // Get the column name from headers colName := expectedTable.Headers[colIdx] - // Apply transformer if one exists for this column if transformer, exists := input.ColumnTransformers[colName]; exists && transformer != nil { transformedRow[colIdx] = transformer(value) } else { @@ -41,40 +39,54 @@ func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqua } } + // Build a set of excluded column positions for actual + excludedIdx := make(map[int]struct{}) + for _, idx := range input.ExcludedColumns { + excludedIdx[idx] = struct{}{} + } + + // Transform actual rows (drop excluded columns) actualInStrings := [][]string{} for _, row := range input.Actual { actualRow := []string{} - for _, column := range row { - actualRow = append(actualRow, column) + for colIdx, value := range row { + if _, skip := excludedIdx[colIdx]; skip { + continue + } + actualRow = append(actualRow, value) } actualInStrings = append(actualInStrings, actualRow) } // Sort both expected and actual data if sort columns are specified if len(input.SortColumns) > 0 { - // Create maps of column name to index for sorting - headerIndexMap := make(map[string]int) - for i, header := range expectedTable.Headers { - headerIndexMap[header] = i + // Build mapping from header name -> index in expected + expectedHeaderMap := make(map[string]int) + for i, h := range expectedTable.Headers { + expectedHeaderMap[h] = i + } + + actualHeaderMap := make(map[string]int) + for i, h := range expectedTable.Headers { + actualHeaderMap[h] = i } - // Create a custom sort function that can sort by multiple columns - sortFunc := func(rows [][]string) func(i, j int) bool { + // Sort function + sortFunc := func(rows [][]string, headerMap map[string]int) func(i, j int) bool { return func(i, j int) bool { for _, colName := range input.SortColumns { - if idx, ok := headerIndexMap[colName]; ok && idx < len(rows[i]) && idx < len(rows[j]) { + if idx, ok := headerMap[colName]; ok && idx < len(rows[i]) && idx < len(rows[j]) { if rows[i][idx] != rows[j][idx] { - return rows[i][idx] < rows[j][idx] + return rows[i][idx] < rows[j][idx] // ascending } } } - return false // If all sort columns are equal, maintain original order + return false } } - // Sort both expected and actual results - sort.SliceStable(expected, sortFunc(expected)) - sort.SliceStable(actualInStrings, sortFunc(actualInStrings)) + sort.SliceStable(expected, sortFunc(expected, expectedHeaderMap)) + sort.SliceStable(actualInStrings, sortFunc(actualInStrings, actualHeaderMap)) } assert.Equal(t, expected, actualInStrings, "Result rows do not match expected markdown table")