From e820e72059b7919b7ec281d111f34dc72e5d1ef7 Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 22:00:43 +0700 Subject: [PATCH 1/7] feat: view list metadata by height --- internal/migrations/001-common-actions.sql | 2651 ++++++++++---------- tests/streams/utils/procedure/execute.go | 43 + tests/streams/utils/procedure/types.go | 11 + 3 files changed, 1420 insertions(+), 1285 deletions(-) diff --git a/internal/migrations/001-common-actions.sql b/internal/migrations/001-common-actions.sql index 03c0f3285..b4f7ec2ae 100644 --- a/internal/migrations/001-common-actions.sql +++ b/internal/migrations/001-common-actions.sql @@ -1,1322 +1,1403 @@ -/** - * create_data_provider: Register new data provider - */ -CREATE OR REPLACE ACTION create_data_provider( - $address TEXT -) PUBLIC { - $lower_caller TEXT := LOWER(@caller); - -- Permission Check: Ensure caller has the 'system:network_writer' role. - $has_permission BOOL := false; - for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { - if $row.wallet = $lower_caller AND $row.is_member { - $has_permission := true; - break; - } - } - if NOT $has_permission { - ERROR('Caller does not have the required system:network_writer role to create data provider.'); - } - - $lower_address TEXT := LOWER($address); - - -- Check if address provided is a valid ethereum address - if NOT check_ethereum_address($lower_address) { - ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $lower_address); - } - - INSERT INTO data_providers (id, address, created_at) - SELECT - COALESCE(MAX(id), 0) + 1, - $lower_address, - @height - FROM data_providers - ON CONFLICT DO NOTHING; -}; - -/** - * create_stream: Creates a new stream with required metadata. - * Validates stream_id format, data provider address, and stream type. - * Sets default metadata including type, owner, visibility, and readonly keys. - */ -CREATE OR REPLACE ACTION create_stream( - $stream_id TEXT, - $stream_type TEXT -) PUBLIC { - -- Delegate to batch implementation for single-stream consistency. - create_streams( ARRAY[$stream_id], ARRAY[$stream_type] ); -}; - -/** - * create_streams: Creates multiple streams at once. - * Validates stream_id format, data provider address, and stream type. - * Sets default metadata including type, owner, visibility, and readonly keys. - */ -CREATE OR REPLACE ACTION create_streams( - $stream_ids TEXT[], - $stream_types TEXT[] -) PUBLIC { - $lower_caller TEXT := LOWER(@caller); - -- Permission Check: Ensure caller has the 'system:network_writer' role. - $has_permission BOOL := false; - for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { - if $row.wallet = $lower_caller AND $row.is_member { - $has_permission := true; - break; - } - } - if NOT $has_permission { - ERROR('Caller does not have the required system:network_writer role to create streams.'); - } - - -- Get caller's address (data provider) first - $data_provider TEXT := $lower_caller; - - -- Check if caller is a valid ethereum address - if NOT check_ethereum_address($data_provider) { - ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $data_provider); - } - - -- Check if stream_ids and stream_types arrays have the same length - if array_length($stream_ids) != array_length($stream_types) { - ERROR('Stream IDs and stream types arrays must have the same length'); - } - - -- Validate stream IDs - for $validation_result in validate_stream_ids_format_batch($stream_ids) { - if NOT $validation_result.is_valid { - ERROR('Invalid stream_id format: ' || $validation_result.stream_id || ' - ' || - $validation_result.error_reason); - } - } - - -- Validate stream types using dedicated private function - for $validation_result in validate_stream_types_batch($stream_types) { - IF $validation_result.error_reason != '' { - ERROR('Invalid stream type at position ' || $validation_result.position || ': ' || - $validation_result.stream_type || ' - ' || $validation_result.error_reason); - } - } - - $base_uuid := uuid_generate_kwil('create_streams_' || @txid); - - -- Get the data provider id - $data_provider_id INT; - $dp_found BOOL := false; - for $data_provider_row in SELECT id - FROM data_providers - WHERE address = $data_provider - LIMIT 1 { - $dp_found := true; - $data_provider_id := $data_provider_row.id; - } - - if $dp_found = false { - ERROR('Data provider not found: ' || $data_provider); - } +-- /** +-- * create_data_provider: Register new data provider +-- */ +-- CREATE OR REPLACE ACTION create_data_provider( +-- $address TEXT +-- ) PUBLIC { +-- $lower_caller TEXT := LOWER(@caller); +-- -- Permission Check: Ensure caller has the 'system:network_writer' role. +-- $has_permission BOOL := false; +-- for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { +-- if $row.wallet = $lower_caller AND $row.is_member { +-- $has_permission := true; +-- break; +-- } +-- } +-- if NOT $has_permission { +-- ERROR('Caller does not have the required system:network_writer role to create data provider.'); +-- } + +-- $lower_address TEXT := LOWER($address); + +-- -- Check if address provided is a valid ethereum address +-- if NOT check_ethereum_address($lower_address) { +-- ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $lower_address); +-- } + +-- INSERT INTO data_providers (id, address, created_at) +-- SELECT +-- COALESCE(MAX(id), 0) + 1, +-- $lower_address, +-- @height +-- FROM data_providers +-- ON CONFLICT DO NOTHING; +-- }; + +-- /** +-- * create_stream: Creates a new stream with required metadata. +-- * Validates stream_id format, data provider address, and stream type. +-- * Sets default metadata including type, owner, visibility, and readonly keys. +-- */ +-- CREATE OR REPLACE ACTION create_stream( +-- $stream_id TEXT, +-- $stream_type TEXT +-- ) PUBLIC { +-- -- Delegate to batch implementation for single-stream consistency. +-- create_streams( ARRAY[$stream_id], ARRAY[$stream_type] ); +-- }; + +-- /** +-- * create_streams: Creates multiple streams at once. +-- * Validates stream_id format, data provider address, and stream type. +-- * Sets default metadata including type, owner, visibility, and readonly keys. +-- */ +-- CREATE OR REPLACE ACTION create_streams( +-- $stream_ids TEXT[], +-- $stream_types TEXT[] +-- ) PUBLIC { +-- $lower_caller TEXT := LOWER(@caller); +-- -- Permission Check: Ensure caller has the 'system:network_writer' role. +-- $has_permission BOOL := false; +-- for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { +-- if $row.wallet = $lower_caller AND $row.is_member { +-- $has_permission := true; +-- break; +-- } +-- } +-- if NOT $has_permission { +-- ERROR('Caller does not have the required system:network_writer role to create streams.'); +-- } + +-- -- Get caller's address (data provider) first +-- $data_provider TEXT := $lower_caller; + +-- -- Check if caller is a valid ethereum address +-- if NOT check_ethereum_address($data_provider) { +-- ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $data_provider); +-- } + +-- -- Check if stream_ids and stream_types arrays have the same length +-- if array_length($stream_ids) != array_length($stream_types) { +-- ERROR('Stream IDs and stream types arrays must have the same length'); +-- } + +-- -- Validate stream IDs +-- for $validation_result in validate_stream_ids_format_batch($stream_ids) { +-- if NOT $validation_result.is_valid { +-- ERROR('Invalid stream_id format: ' || $validation_result.stream_id || ' - ' || +-- $validation_result.error_reason); +-- } +-- } + +-- -- Validate stream types using dedicated private function +-- for $validation_result in validate_stream_types_batch($stream_types) { +-- IF $validation_result.error_reason != '' { +-- ERROR('Invalid stream type at position ' || $validation_result.position || ': ' || +-- $validation_result.stream_type || ' - ' || $validation_result.error_reason); +-- } +-- } + +-- $base_uuid := uuid_generate_kwil('create_streams_' || @txid); + +-- -- Get the data provider id +-- $data_provider_id INT; +-- $dp_found BOOL := false; +-- for $data_provider_row in SELECT id +-- FROM data_providers +-- WHERE address = $data_provider +-- LIMIT 1 { +-- $dp_found := true; +-- $data_provider_id := $data_provider_row.id; +-- } + +-- if $dp_found = false { +-- ERROR('Data provider not found: ' || $data_provider); +-- } - -- Create the streams using UNNEST for optimal performance - INSERT INTO streams (id, data_provider_id, data_provider, stream_id, stream_type, created_at) - SELECT - ROW_NUMBER() OVER (ORDER BY t.stream_id) + COALESCE((SELECT MAX(id) FROM streams), 0) AS id, - $data_provider_id, - $data_provider, - t.stream_id, - t.stream_type, - @height - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type); +-- -- Create the streams using UNNEST for optimal performance +-- INSERT INTO streams (id, data_provider_id, data_provider, stream_id, stream_type, created_at) +-- SELECT +-- ROW_NUMBER() OVER (ORDER BY t.stream_id) + COALESCE((SELECT MAX(id) FROM streams), 0) AS id, +-- $data_provider_id, +-- $data_provider, +-- t.stream_id, +-- t.stream_type, +-- @height +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type); - -- Create metadata for the streams using UNNEST for optimal performance - -- Insert stream_owner metadata - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at, - disabled_at, - stream_ref - ) - SELECT - uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'stream_owner' || '1')::UUID, - 'stream_owner'::TEXT, - NULL::INT, - NULL::NUMERIC(36,18), - NULL::BOOL, - NULL::TEXT, - LOWER($data_provider)::TEXT, - @height, - NULL::INT, - s.id - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) - JOIN data_providers dp ON dp.address = $data_provider - JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - - -- Insert read_visibility metadata - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at, - disabled_at, - stream_ref - ) - SELECT - uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'read_visibility' || '2')::UUID, - 'read_visibility'::TEXT, - 0::INT, - NULL::NUMERIC(36,18), - NULL::BOOL, - NULL::TEXT, - NULL::TEXT, - @height, - NULL::INT, - s.id - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) - JOIN data_providers dp ON dp.address = $data_provider - JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - - -- Insert readonly_key metadata (stream_owner) - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at, - disabled_at, - stream_ref - ) - SELECT - uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '3')::UUID, - 'readonly_key'::TEXT, - NULL::INT, - NULL::NUMERIC(36,18), - NULL::BOOL, - 'stream_owner'::TEXT, - NULL::TEXT, - @height, - NULL::INT, - s.id - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) - JOIN data_providers dp ON dp.address = $data_provider - JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - - -- Insert readonly_key metadata (readonly_key) - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at, - disabled_at, - stream_ref - ) - SELECT - uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '4')::UUID, - 'readonly_key'::TEXT, - NULL::INT, - NULL::NUMERIC(36,18), - NULL::BOOL, - 'readonly_key'::TEXT, - NULL::TEXT, - @height, - NULL::INT, - s.id - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) - JOIN data_providers dp ON dp.address = $data_provider - JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - - -- Insert type metadata - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at, - disabled_at, - stream_ref - ) - SELECT - uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'type' || '5')::UUID, - 'type'::TEXT, - NULL::INT, - NULL::NUMERIC(36,18), - NULL::BOOL, - t.stream_type, - NULL::TEXT, - @height, - NULL::INT, - s.id - FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) - JOIN data_providers dp ON dp.address = $data_provider - JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - -}; - - -/** - * insert_metadata: Adds metadata to a stream. - * Validates caller is stream owner and handles different value types. - * Prevents modification of readonly keys. - */ -CREATE OR REPLACE ACTION insert_metadata( - -- not necessarily the caller is the original deployer of the stream - $data_provider TEXT, - $stream_id TEXT, - $key TEXT, - $value TEXT, - $val_type TEXT -) PUBLIC { - -- Initialize value variables - $value_i INT; - $value_s TEXT; - $value_f DECIMAL(36,18); - $value_b BOOL; - $value_ref TEXT; - $data_provider := LOWER($data_provider); - $lower_caller := LOWER(@caller); +-- -- Create metadata for the streams using UNNEST for optimal performance +-- -- Insert stream_owner metadata +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at, +-- disabled_at, +-- stream_ref +-- ) +-- SELECT +-- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'stream_owner' || '1')::UUID, +-- 'stream_owner'::TEXT, +-- NULL::INT, +-- NULL::NUMERIC(36,18), +-- NULL::BOOL, +-- NULL::TEXT, +-- LOWER($data_provider)::TEXT, +-- @height, +-- NULL::INT, +-- s.id +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) +-- JOIN data_providers dp ON dp.address = $data_provider +-- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +-- -- Insert read_visibility metadata +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at, +-- disabled_at, +-- stream_ref +-- ) +-- SELECT +-- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'read_visibility' || '2')::UUID, +-- 'read_visibility'::TEXT, +-- 0::INT, +-- NULL::NUMERIC(36,18), +-- NULL::BOOL, +-- NULL::TEXT, +-- NULL::TEXT, +-- @height, +-- NULL::INT, +-- s.id +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) +-- JOIN data_providers dp ON dp.address = $data_provider +-- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +-- -- Insert readonly_key metadata (stream_owner) +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at, +-- disabled_at, +-- stream_ref +-- ) +-- SELECT +-- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '3')::UUID, +-- 'readonly_key'::TEXT, +-- NULL::INT, +-- NULL::NUMERIC(36,18), +-- NULL::BOOL, +-- 'stream_owner'::TEXT, +-- NULL::TEXT, +-- @height, +-- NULL::INT, +-- s.id +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) +-- JOIN data_providers dp ON dp.address = $data_provider +-- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +-- -- Insert readonly_key metadata (readonly_key) +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at, +-- disabled_at, +-- stream_ref +-- ) +-- SELECT +-- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '4')::UUID, +-- 'readonly_key'::TEXT, +-- NULL::INT, +-- NULL::NUMERIC(36,18), +-- NULL::BOOL, +-- 'readonly_key'::TEXT, +-- NULL::TEXT, +-- @height, +-- NULL::INT, +-- s.id +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) +-- JOIN data_providers dp ON dp.address = $data_provider +-- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +-- -- Insert type metadata +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at, +-- disabled_at, +-- stream_ref +-- ) +-- SELECT +-- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'type' || '5')::UUID, +-- 'type'::TEXT, +-- NULL::INT, +-- NULL::NUMERIC(36,18), +-- NULL::BOOL, +-- t.stream_type, +-- NULL::TEXT, +-- @height, +-- NULL::INT, +-- s.id +-- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) +-- JOIN data_providers dp ON dp.address = $data_provider +-- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +-- }; + + +-- /** +-- * insert_metadata: Adds metadata to a stream. +-- * Validates caller is stream owner and handles different value types. +-- * Prevents modification of readonly keys. +-- */ +-- CREATE OR REPLACE ACTION insert_metadata( +-- -- not necessarily the caller is the original deployer of the stream +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT, +-- $value TEXT, +-- $val_type TEXT +-- ) PUBLIC { +-- -- Initialize value variables +-- $value_i INT; +-- $value_s TEXT; +-- $value_f DECIMAL(36,18); +-- $value_b BOOL; +-- $value_ref TEXT; +-- $data_provider := LOWER($data_provider); +-- $lower_caller := LOWER(@caller); - -- Check if caller is the stream owner - if !is_stream_owner($data_provider, $stream_id, $lower_caller) { - ERROR('Only stream owner can insert metadata'); - } +-- -- Check if caller is the stream owner +-- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { +-- ERROR('Only stream owner can insert metadata'); +-- } - -- Set the appropriate value based on type - if $val_type = 'int' { - $value_i := $value::INT; - } elseif $val_type = 'string' { - $value_s := $value; - } elseif $val_type = 'bool' { - $value_b := $value::BOOL; - } elseif $val_type = 'ref' { - $value_ref := $value; - } elseif $val_type = 'float' { - $value_f := $value::DECIMAL(36,18); - } else { - ERROR(FORMAT('Unknown type used "%s". Valid types = "float" | "bool" | "int" | "ref" | "string"', $val_type)); - } - - $stream_ref := get_stream_id($data_provider, $stream_id); +-- -- Set the appropriate value based on type +-- if $val_type = 'int' { +-- $value_i := $value::INT; +-- } elseif $val_type = 'string' { +-- $value_s := $value; +-- } elseif $val_type = 'bool' { +-- $value_b := $value::BOOL; +-- } elseif $val_type = 'ref' { +-- $value_ref := $value; +-- } elseif $val_type = 'float' { +-- $value_f := $value::DECIMAL(36,18); +-- } else { +-- ERROR(FORMAT('Unknown type used "%s". Valid types = "float" | "bool" | "int" | "ref" | "string"', $val_type)); +-- } + +-- $stream_ref := get_stream_id($data_provider, $stream_id); - -- Check if the key is read-only - $is_readonly BOOL := false; - for $row in SELECT * FROM metadata - WHERE stream_ref = $stream_ref - AND metadata_key = 'readonly_key' - AND value_s = $key LIMIT 1 { - $is_readonly := true; - } +-- -- Check if the key is read-only +-- $is_readonly BOOL := false; +-- for $row in SELECT * FROM metadata +-- WHERE stream_ref = $stream_ref +-- AND metadata_key = 'readonly_key' +-- AND value_s = $key LIMIT 1 { +-- $is_readonly := true; +-- } - if $is_readonly = true { - ERROR('Cannot insert metadata for read-only key'); - } +-- if $is_readonly = true { +-- ERROR('Cannot insert metadata for read-only key'); +-- } - -- Create deterministic UUID for the metadata record - $uuid_key TEXT := @txid || $key || $value; - $uuid UUID := uuid_generate_kwil($uuid_key); - $current_block INT := @height; - $stream_ref INT := get_stream_id($data_provider, $stream_id); +-- -- Create deterministic UUID for the metadata record +-- $uuid_key TEXT := @txid || $key || $value; +-- $uuid UUID := uuid_generate_kwil($uuid_key); +-- $current_block INT := @height; +-- $stream_ref INT := get_stream_id($data_provider, $stream_id); - -- Insert the metadata - INSERT INTO metadata ( - row_id, - metadata_key, - value_i, - value_f, - value_s, - value_b, - value_ref, - created_at, - stream_ref - ) VALUES ( - $uuid, - $key, - $value_i, - $value_f, - $value_s, - $value_b, - LOWER($value_ref), - $current_block, - $stream_ref - ); -}; - -/** - * disable_metadata: Marks a metadata record as disabled. - * Validates caller is stream owner and prevents disabling readonly keys. - */ -CREATE OR REPLACE ACTION disable_metadata( - -- not necessarily the caller is the original deployer of the stream - $data_provider TEXT, - $stream_id TEXT, - $row_id UUID -) PUBLIC { - $data_provider := LOWER($data_provider); - $lower_caller := LOWER(@caller); - -- Check if caller is the stream owner - if !is_stream_owner($data_provider, $stream_id, $lower_caller) { - ERROR('Only stream owner can disable metadata'); - } +-- -- Insert the metadata +-- INSERT INTO metadata ( +-- row_id, +-- metadata_key, +-- value_i, +-- value_f, +-- value_s, +-- value_b, +-- value_ref, +-- created_at, +-- stream_ref +-- ) VALUES ( +-- $uuid, +-- $key, +-- $value_i, +-- $value_f, +-- $value_s, +-- $value_b, +-- LOWER($value_ref), +-- $current_block, +-- $stream_ref +-- ); +-- }; + +-- /** +-- * disable_metadata: Marks a metadata record as disabled. +-- * Validates caller is stream owner and prevents disabling readonly keys. +-- */ +-- CREATE OR REPLACE ACTION disable_metadata( +-- -- not necessarily the caller is the original deployer of the stream +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $row_id UUID +-- ) PUBLIC { +-- $data_provider := LOWER($data_provider); +-- $lower_caller := LOWER(@caller); +-- -- Check if caller is the stream owner +-- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { +-- ERROR('Only stream owner can disable metadata'); +-- } - $current_block INT := @height; - $found BOOL := false; - $metadata_key TEXT; - $stream_ref INT := get_stream_id($data_provider, $stream_id); +-- $current_block INT := @height; +-- $found BOOL := false; +-- $metadata_key TEXT; +-- $stream_ref INT := get_stream_id($data_provider, $stream_id); - -- Get the metadata key first to avoid nested queries - for $metadata_row in SELECT metadata_key - FROM metadata - WHERE row_id = $row_id - AND stream_ref = $stream_ref - AND disabled_at IS NULL - LIMIT 1 { - $found := true; - $metadata_key := $metadata_row.metadata_key; - } +-- -- Get the metadata key first to avoid nested queries +-- for $metadata_row in SELECT metadata_key +-- FROM metadata +-- WHERE row_id = $row_id +-- AND stream_ref = $stream_ref +-- AND disabled_at IS NULL +-- LIMIT 1 { +-- $found := true; +-- $metadata_key := $metadata_row.metadata_key; +-- } - if $found = false { - ERROR('Metadata record not found'); - } +-- if $found = false { +-- ERROR('Metadata record not found'); +-- } - -- In a separate step, check if the key is read-only - $is_readonly BOOL := false; - for $readonly_row in SELECT * FROM metadata - WHERE stream_ref = $stream_ref - AND metadata_key = 'readonly_key' - AND value_s = $metadata_key LIMIT 1 { - $is_readonly := true; - } +-- -- In a separate step, check if the key is read-only +-- $is_readonly BOOL := false; +-- for $readonly_row in SELECT * FROM metadata +-- WHERE stream_ref = $stream_ref +-- AND metadata_key = 'readonly_key' +-- AND value_s = $metadata_key LIMIT 1 { +-- $is_readonly := true; +-- } - if $is_readonly = true { - ERROR('Cannot disable read-only metadata'); - } +-- if $is_readonly = true { +-- ERROR('Cannot disable read-only metadata'); +-- } - -- Update the metadata to mark it as disabled - UPDATE metadata SET disabled_at = $current_block - WHERE row_id = $row_id - AND stream_ref = $stream_ref; -}; - -/** - * check_stream_id_format: Validates stream ID format (st + 30 alphanumeric chars). - */ -CREATE OR REPLACE ACTION check_stream_id_format( - $stream_id TEXT -) PUBLIC view returns (result BOOL) { - -- Check that the stream_id is exactly 32 characters and starts with "st" - if LENGTH($stream_id) != 32 OR substring($stream_id, 1, 2) != 'st' { - return false; - } - - -- Iterate through each character after the "st" prefix. - for $i in 3..32 { - $c TEXT := substring($stream_id, $i, 1); - if NOT ( - ($c >= '0' AND $c <= '9') - OR ($c >= 'a' AND $c <= 'z') - ) { - return false; - } - } - - return true; -}; - -/** - * validate_stream_ids_format_batch: Validates multiple stream ID formats efficiently. - * Returns all stream IDs with their validation status and error details. - */ -CREATE OR REPLACE ACTION validate_stream_ids_format_batch( - $stream_ids TEXT[] -) PUBLIC view returns table( - stream_id TEXT, - is_valid BOOL, - error_reason TEXT -) { - -- Pure SQL validation using supported functions only - RETURN SELECT - t.stream_id, - CASE - WHEN LENGTH(t.stream_id) != 32 THEN false - WHEN substring(t.stream_id, 1, 2) != 'st' THEN false - -- Check characters 3-32 are lowercase alphanumeric using basic string functions - WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN false - ELSE true - END AS is_valid, - CASE - WHEN LENGTH(t.stream_id) != 32 THEN 'Invalid length (must be 32 characters)' - WHEN substring(t.stream_id, 1, 2) != 'st' THEN 'Must start with "st"' - -- Check characters 3-32 are lowercase alphanumeric using basic string functions - WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN 'Characters 3-32 must be lowercase alphanumeric' - ELSE '' - END AS error_reason - FROM UNNEST($stream_ids) AS t(stream_id); -}; - -/** - * validate_stream_types_batch: Validates multiple stream types efficiently. - * Returns invalid stream types with their positions and error details. - */ -CREATE OR REPLACE ACTION validate_stream_types_batch( - $stream_types TEXT[] -) PRIVATE view returns table( - position INT, - stream_type TEXT, - error_reason TEXT -) { - -- Use CTE with row_number() since WITH ORDINALITY is not supported in Kuneiform - RETURN WITH indexed_types AS ( - SELECT - row_number() OVER () as idx, - stream_type - FROM UNNEST($stream_types) AS t(stream_type) - ) - SELECT - idx as position, - stream_type, - CASE - WHEN stream_type NOT IN ('primitive', 'composed') THEN - 'Stream type must be "primitive" or "composed"' - ELSE '' - END AS error_reason - FROM indexed_types - WHERE stream_type NOT IN ('primitive', 'composed'); -}; - -/** - * check_ethereum_address: Validates Ethereum address format. - */ -CREATE OR REPLACE ACTION check_ethereum_address( - $data_provider TEXT -) PUBLIC view returns (result BOOL) { - -- Verify the address is exactly 42 characters and starts with "0x" - if LENGTH($data_provider) != 42 OR substring($data_provider, 1, 2) != '0x' { - return false; - } - - -- Iterate through each character after the "0x" prefix. - for $i in 3..42 { - $c TEXT := substring($data_provider, $i, 1); - if NOT ( - ($c >= '0' AND $c <= '9') - OR ($c >= 'a' AND $c <= 'f') - OR ($c >= 'A' AND $c <= 'F') - ) { - return false; - } - } - - return true; -}; - -/** - * delete_stream: Removes a stream and all associated data. - * Only stream owner can perform this action. - */ -CREATE OR REPLACE ACTION delete_stream( - -- not necessarily the caller is the original deployer of the stream - $data_provider TEXT, - $stream_id TEXT -) PUBLIC { - $data_provider := LOWER($data_provider); - $lower_caller := LOWER(@caller); - - if !is_stream_owner($data_provider, $stream_id, $lower_caller) { - ERROR('Only stream owner can delete the stream'); - } - - $stream_ref := get_stream_id($data_provider, $stream_id); - - DELETE FROM streams WHERE id = $stream_ref; -}; - -/** - * is_stream_owner: Checks if caller is the owner of a stream. - * Uses stream_owner metadata to determine ownership. - */ -CREATE OR REPLACE ACTION is_stream_owner_core( - $stream_ref INT, - $caller TEXT -) PRIVATE view returns (is_owner BOOL) { - -- Check if the caller is the owner by looking at the latest stream_owner metadata - for $row in SELECT COALESCE( - -- do not use LOWER here, or it will break the index lookup - (SELECT m.value_ref = LOWER($caller) - FROM metadata m - WHERE m.stream_ref = $stream_ref - AND m.metadata_key = 'stream_owner' - AND m.disabled_at IS NULL - ORDER BY m.created_at DESC - LIMIT 1), false - ) as result { - return $row.result; - } - return false; -}; - -CREATE OR REPLACE ACTION is_stream_owner( - $data_provider TEXT, - $stream_id TEXT, - $caller TEXT -) PUBLIC view returns (is_owner BOOL) { - $data_provider := LOWER($data_provider); - $lower_caller := LOWER($caller); - - -- Check if the stream exists (get_stream_id returns NULL if stream doesn't exist) - $stream_ref := get_stream_id($data_provider, $stream_id); - IF $stream_ref IS NULL { - ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); - } - return is_stream_owner_core($stream_ref, $lower_caller); -}; - -/** - * is_stream_owner_batch: Checks if a wallet is the owner of multiple streams. - * Processes arrays of data providers and stream IDs efficiently. - * Returns a table indicating ownership status for each stream. - */ -CREATE OR REPLACE ACTION is_stream_owner_batch( - $data_providers TEXT[], - $stream_ids TEXT[], - $wallet TEXT -) PUBLIC view returns table( - data_provider TEXT, - stream_id TEXT, - is_owner BOOL -) { - -- Lowercase data providers directly using UNNEST for efficiency - - -- Check that arrays have the same length - if array_length($data_providers) != array_length($stream_ids) { - ERROR('Data providers and stream IDs arrays must have the same length'); - } - - -- Check if the wallet is the owner of each stream - for $row in stream_exists_batch($data_providers, $stream_ids) { - if !$row.stream_exists { - ERROR('stream does not exist: data_provider=' || $row.data_provider || ', stream_id=' || $row.stream_id); - } - } - $lowercase_wallet TEXT := LOWER($wallet); - - -- Use UNNEST for optimal performance with direct LOWER operations - RETURN SELECT - t.data_provider, - t.stream_id, - CASE WHEN m.value_ref IS NOT NULL AND m.value_ref = $lowercase_wallet THEN true ELSE false END AS is_owner - FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) - LEFT JOIN ( - SELECT dp.address as data_provider, s.stream_id, latest.value_ref - FROM ( - SELECT md.stream_ref, md.value_ref, - ROW_NUMBER() OVER (PARTITION BY md.stream_ref ORDER BY md.created_at DESC, md.row_id DESC) AS rn - FROM metadata md - WHERE md.metadata_key = 'stream_owner' - AND md.disabled_at IS NULL - ) latest - JOIN streams s ON latest.stream_ref = s.id - JOIN data_providers dp ON s.data_provider_id = dp.id - WHERE latest.rn = 1 - ) m ON LOWER(t.data_provider) = m.data_provider AND t.stream_id = m.stream_id; -}; - -/** - * is_primitive_stream: Determines if a stream is primitive or composed. - */ -CREATE OR REPLACE ACTION is_primitive_stream( - $data_provider TEXT, - $stream_id TEXT -) PUBLIC view returns (is_primitive BOOL) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - for $row in SELECT stream_type FROM streams - WHERE id = $stream_ref LIMIT 1 { - return $row.stream_type = 'primitive'; - } +-- -- Update the metadata to mark it as disabled +-- UPDATE metadata SET disabled_at = $current_block +-- WHERE row_id = $row_id +-- AND stream_ref = $stream_ref; +-- }; + +-- /** +-- * check_stream_id_format: Validates stream ID format (st + 30 alphanumeric chars). +-- */ +-- CREATE OR REPLACE ACTION check_stream_id_format( +-- $stream_id TEXT +-- ) PUBLIC view returns (result BOOL) { +-- -- Check that the stream_id is exactly 32 characters and starts with "st" +-- if LENGTH($stream_id) != 32 OR substring($stream_id, 1, 2) != 'st' { +-- return false; +-- } + +-- -- Iterate through each character after the "st" prefix. +-- for $i in 3..32 { +-- $c TEXT := substring($stream_id, $i, 1); +-- if NOT ( +-- ($c >= '0' AND $c <= '9') +-- OR ($c >= 'a' AND $c <= 'z') +-- ) { +-- return false; +-- } +-- } + +-- return true; +-- }; + +-- /** +-- * validate_stream_ids_format_batch: Validates multiple stream ID formats efficiently. +-- * Returns all stream IDs with their validation status and error details. +-- */ +-- CREATE OR REPLACE ACTION validate_stream_ids_format_batch( +-- $stream_ids TEXT[] +-- ) PUBLIC view returns table( +-- stream_id TEXT, +-- is_valid BOOL, +-- error_reason TEXT +-- ) { +-- -- Pure SQL validation using supported functions only +-- RETURN SELECT +-- t.stream_id, +-- CASE +-- WHEN LENGTH(t.stream_id) != 32 THEN false +-- WHEN substring(t.stream_id, 1, 2) != 'st' THEN false +-- -- Check characters 3-32 are lowercase alphanumeric using basic string functions +-- WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN false +-- ELSE true +-- END AS is_valid, +-- CASE +-- WHEN LENGTH(t.stream_id) != 32 THEN 'Invalid length (must be 32 characters)' +-- WHEN substring(t.stream_id, 1, 2) != 'st' THEN 'Must start with "st"' +-- -- Check characters 3-32 are lowercase alphanumeric using basic string functions +-- WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN 'Characters 3-32 must be lowercase alphanumeric' +-- ELSE '' +-- END AS error_reason +-- FROM UNNEST($stream_ids) AS t(stream_id); +-- }; + +-- /** +-- * validate_stream_types_batch: Validates multiple stream types efficiently. +-- * Returns invalid stream types with their positions and error details. +-- */ +-- CREATE OR REPLACE ACTION validate_stream_types_batch( +-- $stream_types TEXT[] +-- ) PRIVATE view returns table( +-- position INT, +-- stream_type TEXT, +-- error_reason TEXT +-- ) { +-- -- Use CTE with row_number() since WITH ORDINALITY is not supported in Kuneiform +-- RETURN WITH indexed_types AS ( +-- SELECT +-- row_number() OVER () as idx, +-- stream_type +-- FROM UNNEST($stream_types) AS t(stream_type) +-- ) +-- SELECT +-- idx as position, +-- stream_type, +-- CASE +-- WHEN stream_type NOT IN ('primitive', 'composed') THEN +-- 'Stream type must be "primitive" or "composed"' +-- ELSE '' +-- END AS error_reason +-- FROM indexed_types +-- WHERE stream_type NOT IN ('primitive', 'composed'); +-- }; + +-- /** +-- * check_ethereum_address: Validates Ethereum address format. +-- */ +-- CREATE OR REPLACE ACTION check_ethereum_address( +-- $data_provider TEXT +-- ) PUBLIC view returns (result BOOL) { +-- -- Verify the address is exactly 42 characters and starts with "0x" +-- if LENGTH($data_provider) != 42 OR substring($data_provider, 1, 2) != '0x' { +-- return false; +-- } + +-- -- Iterate through each character after the "0x" prefix. +-- for $i in 3..42 { +-- $c TEXT := substring($data_provider, $i, 1); +-- if NOT ( +-- ($c >= '0' AND $c <= '9') +-- OR ($c >= 'a' AND $c <= 'f') +-- OR ($c >= 'A' AND $c <= 'F') +-- ) { +-- return false; +-- } +-- } + +-- return true; +-- }; + +-- /** +-- * delete_stream: Removes a stream and all associated data. +-- * Only stream owner can perform this action. +-- */ +-- CREATE OR REPLACE ACTION delete_stream( +-- -- not necessarily the caller is the original deployer of the stream +-- $data_provider TEXT, +-- $stream_id TEXT +-- ) PUBLIC { +-- $data_provider := LOWER($data_provider); +-- $lower_caller := LOWER(@caller); + +-- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { +-- ERROR('Only stream owner can delete the stream'); +-- } + +-- $stream_ref := get_stream_id($data_provider, $stream_id); + +-- DELETE FROM streams WHERE id = $stream_ref; +-- }; + +-- /** +-- * is_stream_owner: Checks if caller is the owner of a stream. +-- * Uses stream_owner metadata to determine ownership. +-- */ +-- CREATE OR REPLACE ACTION is_stream_owner_core( +-- $stream_ref INT, +-- $caller TEXT +-- ) PRIVATE view returns (is_owner BOOL) { +-- -- Check if the caller is the owner by looking at the latest stream_owner metadata +-- for $row in SELECT COALESCE( +-- -- do not use LOWER here, or it will break the index lookup +-- (SELECT m.value_ref = LOWER($caller) +-- FROM metadata m +-- WHERE m.stream_ref = $stream_ref +-- AND m.metadata_key = 'stream_owner' +-- AND m.disabled_at IS NULL +-- ORDER BY m.created_at DESC +-- LIMIT 1), false +-- ) as result { +-- return $row.result; +-- } +-- return false; +-- }; + +-- CREATE OR REPLACE ACTION is_stream_owner( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $caller TEXT +-- ) PUBLIC view returns (is_owner BOOL) { +-- $data_provider := LOWER($data_provider); +-- $lower_caller := LOWER($caller); + +-- -- Check if the stream exists (get_stream_id returns NULL if stream doesn't exist) +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- IF $stream_ref IS NULL { +-- ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); +-- } +-- return is_stream_owner_core($stream_ref, $lower_caller); +-- }; + +-- /** +-- * is_stream_owner_batch: Checks if a wallet is the owner of multiple streams. +-- * Processes arrays of data providers and stream IDs efficiently. +-- * Returns a table indicating ownership status for each stream. +-- */ +-- CREATE OR REPLACE ACTION is_stream_owner_batch( +-- $data_providers TEXT[], +-- $stream_ids TEXT[], +-- $wallet TEXT +-- ) PUBLIC view returns table( +-- data_provider TEXT, +-- stream_id TEXT, +-- is_owner BOOL +-- ) { +-- -- Lowercase data providers directly using UNNEST for efficiency + +-- -- Check that arrays have the same length +-- if array_length($data_providers) != array_length($stream_ids) { +-- ERROR('Data providers and stream IDs arrays must have the same length'); +-- } + +-- -- Check if the wallet is the owner of each stream +-- for $row in stream_exists_batch($data_providers, $stream_ids) { +-- if !$row.stream_exists { +-- ERROR('stream does not exist: data_provider=' || $row.data_provider || ', stream_id=' || $row.stream_id); +-- } +-- } +-- $lowercase_wallet TEXT := LOWER($wallet); + +-- -- Use UNNEST for optimal performance with direct LOWER operations +-- RETURN SELECT +-- t.data_provider, +-- t.stream_id, +-- CASE WHEN m.value_ref IS NOT NULL AND m.value_ref = $lowercase_wallet THEN true ELSE false END AS is_owner +-- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) +-- LEFT JOIN ( +-- SELECT dp.address as data_provider, s.stream_id, latest.value_ref +-- FROM ( +-- SELECT md.stream_ref, md.value_ref, +-- ROW_NUMBER() OVER (PARTITION BY md.stream_ref ORDER BY md.created_at DESC, md.row_id DESC) AS rn +-- FROM metadata md +-- WHERE md.metadata_key = 'stream_owner' +-- AND md.disabled_at IS NULL +-- ) latest +-- JOIN streams s ON latest.stream_ref = s.id +-- JOIN data_providers dp ON s.data_provider_id = dp.id +-- WHERE latest.rn = 1 +-- ) m ON LOWER(t.data_provider) = m.data_provider AND t.stream_id = m.stream_id; +-- }; + +-- /** +-- * is_primitive_stream: Determines if a stream is primitive or composed. +-- */ +-- CREATE OR REPLACE ACTION is_primitive_stream( +-- $data_provider TEXT, +-- $stream_id TEXT +-- ) PUBLIC view returns (is_primitive BOOL) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- for $row in SELECT stream_type FROM streams +-- WHERE id = $stream_ref LIMIT 1 { +-- return $row.stream_type = 'primitive'; +-- } + +-- ERROR('Stream not found: data_provider=' || $data_provider || ' stream_id=' || $stream_id); +-- }; + +-- /** +-- * is_primitive_stream_batch: Checks if multiple streams are primitive in a single query. +-- * Returns a table with primitive status for each stream. +-- * Only checks streams that exist - does not error on non-existent streams. +-- */ +-- CREATE OR REPLACE ACTION is_primitive_stream_batch( +-- $data_providers TEXT[], +-- $stream_ids TEXT[] +-- ) PUBLIC view returns table( +-- data_provider TEXT, +-- stream_id TEXT, +-- is_primitive BOOL +-- ) { +-- -- Lowercase data providers directly using UNNEST for efficiency + +-- -- Check that arrays have the same length +-- if array_length($data_providers) != array_length($stream_ids) { +-- ERROR('Data providers and stream IDs arrays must have the same length'); +-- } + +-- -- Use UNNEST for optimal performance with direct LOWER operations +-- RETURN SELECT +-- t.data_provider, +-- t.stream_id, +-- COALESCE(s.stream_type = 'primitive', false) AS is_primitive +-- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) +-- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) +-- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; +-- }; + +-- /** +-- * get_metadata: Retrieves metadata for a stream with pagination and filtering. +-- * Supports ordering by creation time and filtering by key and reference. +-- */ +-- CREATE OR REPLACE ACTION get_metadata_core( +-- $stream_ref INT, +-- $key TEXT, +-- $ref TEXT, +-- $limit INT, +-- $offset INT, +-- $order_by TEXT +-- ) PRIVATE view returns table( +-- row_id uuid, +-- value_i int, +-- value_f NUMERIC(36,18), +-- value_b bool, +-- value_s TEXT, +-- value_ref TEXT, +-- created_at INT +-- ) { +-- -- Set default values if parameters are null +-- if $limit IS NULL { +-- $limit := 100; +-- } +-- if $offset IS NULL { +-- $offset := 0; +-- } +-- if $order_by IS NULL { +-- $order_by := 'created_at DESC'; +-- } + +-- RETURN SELECT row_id, +-- value_i, +-- value_f, +-- value_b, +-- value_s, +-- value_ref, +-- created_at +-- FROM metadata +-- WHERE metadata_key = $key +-- AND disabled_at IS NULL +-- -- do not use LOWER on value_ref, or it will break the index lookup +-- AND ($ref IS NULL OR value_ref = LOWER($ref)) +-- AND stream_ref = $stream_ref +-- ORDER BY +-- CASE WHEN $order_by = 'created_at DESC' THEN created_at END DESC, +-- CASE WHEN $order_by = 'created_at ASC' THEN created_at END ASC +-- LIMIT $limit OFFSET $offset; +-- }; + +-- CREATE OR REPLACE ACTION get_metadata( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT, +-- $ref TEXT, +-- $limit INT, +-- $offset INT, +-- $order_by TEXT +-- ) PUBLIC view returns table( +-- row_id uuid, +-- value_i int, +-- value_f NUMERIC(36,18), +-- value_b bool, +-- value_s TEXT, +-- value_ref TEXT, +-- created_at INT +-- ) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- for $row in get_metadata_core($stream_ref, $key, $ref, $limit, $offset, $order_by) { +-- RETURN NEXT $row.row_id, $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref, $row.created_at; +-- } +-- }; + +-- -- Compatibility wrapper that returns single value from get_metadata_core (for functions expecting single value) +-- CREATE OR REPLACE ACTION get_metadata_priv_single( +-- $stream_ref INT, +-- $key TEXT, +-- $ref TEXT +-- ) PRIVATE view returns (value TEXT) { +-- $result TEXT := NULL; +-- for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { +-- $result := $row.value_ref; +-- } +-- RETURN $result; +-- }; + +-- /** +-- * get_latest_metadata: Retrieves the latest metadata for a stream. +-- */ +-- CREATE OR REPLACE ACTION get_latest_metadata_core( +-- $stream_ref INT, +-- $key TEXT, +-- $ref TEXT +-- ) PRIVATE view returns table( +-- value_i INT, +-- value_f NUMERIC(36,18), +-- value_b BOOL, +-- value_s TEXT, +-- value_ref TEXT +-- ) { +-- for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { +-- RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; +-- } +-- }; + +-- CREATE OR REPLACE ACTION get_latest_metadata( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT, +-- $ref TEXT +-- ) PUBLIC view returns table( +-- value_i INT, +-- value_f NUMERIC(36,18), +-- value_b BOOL, +-- value_s TEXT, +-- value_ref TEXT +-- ) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); + +-- for $row in get_latest_metadata_core($stream_ref, $key, $ref) { +-- RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; +-- } +-- }; + +-- /** +-- * get_latest_metadata_int: Retrieves the latest metadata value for a stream. +-- */ +-- CREATE OR REPLACE ACTION get_latest_metadata_int_core( +-- $stream_ref INT, +-- $key TEXT +-- ) PRIVATE view returns (value INT) { +-- $result INT := NULL; +-- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { +-- $result := $row.value_i; +-- } +-- RETURN $result; +-- }; + +-- CREATE OR REPLACE ACTION get_latest_metadata_int( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT +-- ) PUBLIC view returns (value INT) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- return get_latest_metadata_int_core($stream_ref, $key); +-- }; + +-- /** +-- * get_latest_metadata_ref: Retrieves the latest metadata value for a stream. +-- */ +-- CREATE OR REPLACE ACTION get_latest_metadata_ref_core( +-- $stream_ref INT, +-- $key TEXT, +-- $ref TEXT +-- ) PRIVATE view returns (value TEXT) { +-- $result TEXT := NULL; +-- for $row in get_latest_metadata_core($stream_ref, $key, $ref) { +-- $result := $row.value_ref; +-- } +-- RETURN $result; +-- }; + +-- CREATE OR REPLACE ACTION get_latest_metadata_ref( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT, +-- $ref TEXT +-- ) PUBLIC view returns (value TEXT) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- return get_latest_metadata_ref_core($stream_ref, $key, $ref); +-- }; + +-- /** +-- * get_latest_metadata_bool: Retrieves the latest metadata value for a stream. +-- */ +-- CREATE OR REPLACE ACTION get_latest_metadata_bool_core( +-- $stream_ref INT, +-- $key TEXT +-- ) PRIVATE view returns (value BOOL) { +-- $result BOOL := NULL; +-- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { +-- $result := $row.value_b; +-- } +-- RETURN $result; +-- }; + +-- CREATE OR REPLACE ACTION get_latest_metadata_bool( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT +-- ) PUBLIC view returns (value BOOL) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- return get_latest_metadata_bool_core($stream_ref, $key); +-- }; + +-- /** +-- * get_latest_metadata_string: Retrieves the latest metadata value for a stream. +-- */ +-- CREATE OR REPLACE ACTION get_latest_metadata_string_core( +-- $stream_ref INT, +-- $key TEXT +-- ) PRIVATE view returns (value TEXT) { +-- $result TEXT := NULL; +-- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { +-- $result := $row.value_s; +-- } +-- RETURN $result; +-- }; + +-- CREATE OR REPLACE ACTION get_latest_metadata_string( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $key TEXT +-- ) PUBLIC view returns (value TEXT) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- return get_latest_metadata_string_core($stream_ref, $key); +-- }; + +-- /** +-- * get_category_streams: Retrieves all streams in a category (composed stream). +-- * For primitive streams, returns just the stream itself. +-- * For composed streams, recursively traverses taxonomy to find all substreams. +-- * It doesn't check for the existence of the substreams, it just returns them. +-- */ +-- CREATE OR REPLACE ACTION get_category_streams( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $active_from INT, +-- $active_to INT +-- ) PUBLIC view returns table(data_provider TEXT, stream_id TEXT) { +-- $data_provider := LOWER($data_provider); + +-- -- Check if stream exists (get_stream_id returns NULL if stream doesn't exist) +-- $stream_ref := get_stream_id($data_provider, $stream_id); +-- IF $stream_ref IS NULL { +-- ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); +-- } + +-- -- Always return itself first +-- RETURN NEXT $data_provider, $stream_id; + +-- -- For primitive streams, just return the stream itself +-- if is_primitive_stream($data_provider, $stream_id) == true { +-- RETURN; +-- } + +-- -- Set boundaries for time intervals +-- $max_int8 INT := 9223372036854775000; +-- $effective_active_from INT := COALESCE($active_from, 0); +-- $effective_active_to INT := COALESCE($active_to, $max_int8); + +-- -- Get all substreams with proper recursive traversal +-- return WITH RECURSIVE substreams AS ( +-- /*------------------------------------------------------------------ +-- * (1) Base Case: overshadow logic for ($data_provider, $stream_id). +-- * - For each distinct start_time, pick the row with the max group_sequence. +-- * - next_start is used to define [group_sequence_start, group_sequence_end]. +-- *------------------------------------------------------------------*/ +-- SELECT +-- base.stream_ref, +-- base.child_stream_ref, + +-- -- The interval during which this row is active: +-- base.start_time AS group_sequence_start, +-- COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end +-- FROM ( +-- -- Find rows with maximum group_sequence for each start_time +-- SELECT +-- t.stream_ref, +-- t.child_stream_ref, +-- t.start_time, +-- t.group_sequence, +-- MAX(t.group_sequence) OVER ( +-- PARTITION BY t.stream_ref, t.start_time +-- ) AS max_group_sequence +-- FROM taxonomies t +-- WHERE t.stream_ref = $stream_ref +-- AND t.disabled_at IS NULL +-- AND t.start_time <= $effective_active_to +-- AND t.start_time >= COALESCE(( +-- -- Find the most recent taxonomy at or before effective_active_from +-- SELECT t2.start_time +-- FROM taxonomies t2 +-- WHERE t2.stream_ref = t.stream_ref +-- AND t2.disabled_at IS NULL +-- AND t2.start_time <= $effective_active_from +-- ORDER BY t2.start_time DESC, t2.group_sequence DESC +-- LIMIT 1 +-- ), 0 +-- ) +-- ) base +-- JOIN ( +-- /* Distinct start_times for top-level (dp, sid), used for LEAD() */ +-- SELECT +-- dt.stream_ref, +-- dt.start_time, +-- LEAD(dt.start_time) OVER ( +-- PARTITION BY dt.stream_ref +-- ORDER BY dt.start_time +-- ) AS next_start +-- FROM ( +-- SELECT DISTINCT +-- t.stream_ref, +-- t.start_time +-- FROM taxonomies t +-- WHERE t.stream_ref = $stream_ref +-- AND t.disabled_at IS NULL +-- AND t.start_time <= $effective_active_to +-- AND t.start_time >= COALESCE(( +-- SELECT t2.start_time +-- FROM taxonomies t2 +-- WHERE t2.stream_ref = t.stream_ref +-- AND t2.disabled_at IS NULL +-- AND t2.start_time <= $effective_active_from +-- ORDER BY t2.start_time DESC, t2.group_sequence DESC +-- LIMIT 1 +-- ), 0 +-- ) +-- ) dt +-- ) ot +-- ON base.stream_ref = ot.stream_ref +-- AND base.start_time = ot.start_time +-- WHERE base.group_sequence = base.max_group_sequence + +-- UNION + +-- /*------------------------------------------------------------------ +-- * (2) Recursive Child-Level Overshadow: +-- * For each discovered child, gather overshadow rows for that child +-- * and produce intervals that overlap the parent's own active interval. +-- *------------------------------------------------------------------*/ +-- SELECT +-- parent.stream_ref, +-- child.child_stream_ref, + +-- -- Intersection of parent's active interval and child's: +-- GREATEST(parent.group_sequence_start, child.start_time) AS group_sequence_start, +-- LEAST(parent.group_sequence_end, child.group_sequence_end) AS group_sequence_end +-- FROM substreams parent +-- JOIN ( +-- /* Child overshadow logic, same pattern as above but for child dp/sid. */ +-- SELECT +-- base.stream_ref, +-- base.child_stream_ref, +-- base.start_time, +-- COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end +-- FROM ( +-- SELECT +-- t.stream_ref, +-- t.child_stream_ref, +-- t.start_time, +-- t.group_sequence, +-- MAX(t.group_sequence) OVER ( +-- PARTITION BY t.stream_ref, t.start_time +-- ) AS max_group_sequence +-- FROM taxonomies t +-- WHERE t.disabled_at IS NULL +-- AND t.start_time <= $effective_active_to +-- AND t.start_time >= COALESCE(( +-- -- Most recent taxonomy at or before effective_from +-- SELECT t2.start_time +-- FROM taxonomies t2 +-- WHERE t2.stream_ref = t.stream_ref +-- AND t2.disabled_at IS NULL +-- AND t2.start_time <= $effective_active_from +-- ORDER BY t2.start_time DESC, t2.group_sequence DESC +-- LIMIT 1 +-- ), 0 +-- ) +-- ) base +-- JOIN ( +-- /* Distinct start_times at child level */ +-- SELECT +-- dt.stream_ref, +-- dt.start_time, +-- LEAD(dt.start_time) OVER ( +-- PARTITION BY dt.stream_ref +-- ORDER BY dt.start_time +-- ) AS next_start +-- FROM ( +-- SELECT DISTINCT +-- t.stream_ref, +-- t.start_time +-- FROM taxonomies t +-- WHERE t.disabled_at IS NULL +-- AND t.start_time <= $effective_active_to +-- AND t.start_time >= COALESCE(( +-- SELECT t2.start_time +-- FROM taxonomies t2 +-- WHERE t2.stream_ref = t.stream_ref +-- AND t2.disabled_at IS NULL +-- AND t2.start_time <= $effective_active_from +-- ORDER BY t2.start_time DESC, t2.group_sequence DESC +-- LIMIT 1 +-- ), 0 +-- ) +-- ) dt +-- ) ot +-- ON base.stream_ref = ot.stream_ref +-- AND base.start_time = ot.start_time +-- WHERE base.group_sequence = base.max_group_sequence +-- ) child +-- ON child.stream_ref = parent.child_stream_ref + +-- /* Overlap check: child's interval must intersect parent's */ +-- WHERE child.start_time <= parent.group_sequence_end +-- AND child.group_sequence_end >= parent.group_sequence_start +-- ) +-- SELECT DISTINCT +-- child_dp.address as child_data_provider, +-- child_s.stream_id as child_stream_id +-- FROM substreams sub +-- JOIN streams child_s ON sub.child_stream_ref = child_s.id +-- JOIN data_providers child_dp ON child_s.data_provider_id = child_dp.id; +-- }; + +-- /** +-- * stream_exists: Simple check if a stream exists in the database. +-- */ +-- CREATE OR REPLACE ACTION stream_exists( +-- $data_provider TEXT, +-- $stream_id TEXT +-- ) PUBLIC view returns (result BOOL) { +-- $data_provider := LOWER($data_provider); +-- $stream_ref := get_stream_id($data_provider, $stream_id); + +-- for $row in SELECT 1 FROM streams WHERE id = $stream_ref { +-- return true; +-- } +-- return false; +-- }; + +-- /** +-- * stream_exists_batch_core: Private version that uses stream refs directly. +-- * Checks if multiple streams exist using their stream references. +-- * Returns false if any stream refs are null (indicating non-existent streams). +-- * Returns true only if all streams exist and no stream refs are null. +-- */ +-- CREATE OR REPLACE ACTION stream_exists_batch_core( +-- $stream_refs INT[] +-- ) PRIVATE VIEW RETURNS (result BOOL) { +-- -- Use UNNEST for efficient batch processing +-- for $row in SELECT CASE +-- WHEN EXISTS ( +-- SELECT 1 FROM UNNEST($stream_refs) AS t(stream_ref) WHERE t.stream_ref IS NULL +-- ) THEN false +-- ELSE NOT EXISTS ( +-- SELECT 1 +-- FROM UNNEST($stream_refs) AS t(stream_ref) +-- LEFT JOIN streams s ON s.id = t.stream_ref +-- WHERE s.id IS NULL +-- AND t.stream_ref IS NOT NULL +-- ) +-- END AS result +-- FROM (SELECT 1) dummy { +-- return $row.result; +-- } +-- return false; +-- }; + +-- /** +-- * is_primitive_stream_batch_core: Private version that uses stream refs directly. +-- * Checks if multiple streams are primitive using their stream references. +-- * Returns false if any stream refs are null (indicating non-existent streams). +-- * Returns true only if all streams exist and are primitive. +-- */ +-- CREATE OR REPLACE ACTION is_primitive_stream_batch_core( +-- $stream_refs INT[] +-- ) PRIVATE VIEW RETURNS (result BOOL) { +-- -- Use UNNEST for optimal performance - direct array processing without recursion +-- for $row in SELECT CASE +-- WHEN EXISTS ( +-- SELECT 1 +-- FROM UNNEST($stream_refs) AS t(stream_ref) +-- WHERE t.stream_ref IS NULL +-- ) THEN false +-- ELSE NOT EXISTS ( +-- SELECT 1 +-- FROM UNNEST($stream_refs) AS t(stream_ref) +-- JOIN streams s ON s.id = t.stream_ref +-- WHERE s.stream_type != 'primitive' +-- AND t.stream_ref IS NOT NULL +-- ) +-- END AS result +-- FROM (SELECT 1) dummy { +-- return $row.result; +-- } +-- return false; +-- }; + +-- /** +-- * stream_exists_batch: Checks existence of multiple streams in a single query. +-- * Returns a table with existence status for each stream. +-- */ +-- CREATE OR REPLACE ACTION stream_exists_batch( +-- $data_providers TEXT[], +-- $stream_ids TEXT[] +-- ) PUBLIC view returns table( +-- data_provider TEXT, +-- stream_id TEXT, +-- stream_exists BOOL +-- ) { +-- -- Lowercase data providers directly using UNNEST for efficiency + +-- -- Check that arrays have the same length +-- if array_length($data_providers) != array_length($stream_ids) { +-- ERROR('Data providers and stream IDs arrays must have the same length'); +-- } + +-- -- Use UNNEST for optimal performance with direct LOWER operations +-- RETURN SELECT +-- t.data_provider, +-- t.stream_id, +-- CASE WHEN s.data_provider IS NOT NULL THEN true ELSE false END AS stream_exists +-- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) +-- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) +-- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; +-- }; + +-- CREATE OR REPLACE ACTION transfer_stream_ownership( +-- $data_provider TEXT, +-- $stream_id TEXT, +-- $new_owner TEXT +-- ) PUBLIC { +-- $data_provider := LOWER($data_provider); +-- $new_owner := LOWER($new_owner); +-- $lower_caller := LOWER(@caller); +-- $stream_ref := get_stream_id($data_provider, $stream_id); + +-- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { +-- ERROR('Only stream owner can transfer ownership'); +-- } + +-- -- Check if new owner is a valid ethereum address +-- if NOT check_ethereum_address($new_owner) { +-- ERROR('Invalid new owner address. Must be a valid Ethereum address: ' || $new_owner); +-- } + +-- -- Update the stream_owner metadata +-- UPDATE metadata SET value_ref = LOWER($new_owner) +-- WHERE metadata_key = 'stream_owner' +-- AND stream_ref = $stream_ref; +-- }; + +-- /** +-- * filter_streams_by_existence: Filters streams based on existence. +-- * Can return either existing or non-existing streams based on existing_only flag. +-- * Takes arrays of data providers and stream IDs as input. +-- * Uses efficient WITH RECURSIVE pattern for batch processing. +-- */ +-- CREATE OR REPLACE ACTION filter_streams_by_existence( +-- $data_providers TEXT[], +-- $stream_ids TEXT[], +-- $existing_only BOOL +-- ) PUBLIC view returns table( +-- data_provider TEXT, +-- stream_id TEXT +-- ) { +-- -- Lowercase data providers directly using UNNEST for efficiency + +-- -- default to return existing streams +-- if $existing_only IS NULL { +-- $existing_only := true; +-- } - ERROR('Stream not found: data_provider=' || $data_provider || ' stream_id=' || $stream_id); -}; +-- -- Check that arrays have the same length +-- if array_length($data_providers) != array_length($stream_ids) { +-- ERROR('Data providers and stream IDs arrays must have the same length'); +-- } + +-- -- Use UNNEST for efficient batch processing +-- RETURN SELECT +-- t.data_provider, +-- t.stream_id +-- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) +-- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) +-- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id +-- WHERE (CASE WHEN s.id IS NOT NULL THEN true ELSE false END) = $existing_only; +-- }; + +-- CREATE OR REPLACE ACTION list_streams( +-- $data_provider TEXT, +-- $limit INT, +-- $offset INT, +-- $order_by TEXT, +-- $block_height INT +-- ) PUBLIC view returns table( +-- data_provider TEXT, +-- stream_id TEXT, +-- stream_type TEXT, +-- created_at INT8 +-- ) { +-- $data_provider := LOWER($data_provider); + +-- if $limit > 5000 { +-- ERROR('Limit exceeds maximum allowed value of 5000'); +-- } +-- if $limit IS NULL { +-- $limit := 5000; +-- } +-- if $limit == 0 { +-- $limit := 5000; +-- } +-- if $offset IS NULL { +-- $offset := 0; +-- } +-- if $order_by IS NULL { +-- $order_by := 'created_at DESC'; +-- } +-- if $order_by == '' { +-- $order_by := 'created_at DESC'; +-- } + +-- RETURN SELECT +-- dp.address as data_provider, +-- s.stream_id, +-- s.stream_type, +-- s.created_at +-- FROM streams s +-- JOIN data_providers dp ON s.data_provider_id = dp.id +-- -- do not use LOWER on dp.address, or it will break the index lookup +-- WHERE ($data_provider IS NULL OR $data_provider = '' OR dp.address = LOWER($data_provider)) +-- AND s.created_at > $block_height +-- ORDER BY +-- CASE WHEN $order_by = 'created_at DESC' THEN s.created_at END DESC, +-- CASE WHEN $order_by = 'created_at ASC' THEN s.created_at END ASC, +-- CASE WHEN $order_by = 'stream_id ASC' THEN stream_id END ASC, +-- CASE WHEN $order_by = 'stream_id DESC' THEN stream_id END DESC, +-- CASE WHEN $order_by = 'stream_type ASC' THEN stream_type END ASC, +-- CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC +-- LIMIT $limit OFFSET $offset; +-- }; /** - * is_primitive_stream_batch: Checks if multiple streams are primitive in a single query. - * Returns a table with primitive status for each stream. - * Only checks streams that exist - does not error on non-existent streams. + * list_metadata_by_height: Queries metadata within a specific block height range. + * Supports pagination. + * Supports filtering by key and ref. + * + * Parameters: + * $key: filter by metadata key. + *. $ref: filter by value reference. + * $from_height: Start height (inclusive). If NULL, uses earliest available. + * $to_height: End height (inclusive). If NULL, uses current height. + * $limit: Maximum number of results to return. + * $offset: Number of results to skip for pagination. + * + * Returns: + * Table with metadata entries matching the criteria. */ -CREATE OR REPLACE ACTION is_primitive_stream_batch( - $data_providers TEXT[], - $stream_ids TEXT[] +CREATE OR REPLACE ACTION list_metadata_by_height( + $key TEXT, + $ref TEXT, + $from_height INT8, + $to_height INT8, + $limit INT, + $offset INT ) PUBLIC view returns table( - data_provider TEXT, - stream_id TEXT, - is_primitive BOOL -) { - -- Lowercase data providers directly using UNNEST for efficiency - - -- Check that arrays have the same length - if array_length($data_providers) != array_length($stream_ids) { - ERROR('Data providers and stream IDs arrays must have the same length'); - } - - -- Use UNNEST for optimal performance with direct LOWER operations - RETURN SELECT - t.data_provider, - t.stream_id, - COALESCE(s.stream_type = 'primitive', false) AS is_primitive - FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) - LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) - LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; -}; - -/** - * get_metadata: Retrieves metadata for a stream with pagination and filtering. - * Supports ordering by creation time and filtering by key and reference. - */ -CREATE OR REPLACE ACTION get_metadata_core( - $stream_ref INT, - $key TEXT, - $ref TEXT, - $limit INT, - $offset INT, - $order_by TEXT -) PRIVATE view returns table( + stream_ref INT, row_id uuid, - value_i int, + value_i INT, value_f NUMERIC(36,18), value_b bool, value_s TEXT, value_ref TEXT, - created_at INT + created_at INT8 ) { - -- Set default values if parameters are null + -- Set defaults for pagination and validate values if $limit IS NULL { - $limit := 100; + $limit := 1000; } if $offset IS NULL { $offset := 0; } - if $order_by IS NULL { - $order_by := 'created_at DESC'; - } - - RETURN SELECT row_id, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at - FROM metadata - WHERE metadata_key = $key - AND disabled_at IS NULL - -- do not use LOWER on value_ref, or it will break the index lookup - AND ($ref IS NULL OR value_ref = LOWER($ref)) - AND stream_ref = $stream_ref - ORDER BY - CASE WHEN $order_by = 'created_at DESC' THEN created_at END DESC, - CASE WHEN $order_by = 'created_at ASC' THEN created_at END ASC - LIMIT $limit OFFSET $offset; -}; - -CREATE OR REPLACE ACTION get_metadata( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT, - $ref TEXT, - $limit INT, - $offset INT, - $order_by TEXT -) PUBLIC view returns table( - row_id uuid, - value_i int, - value_f NUMERIC(36,18), - value_b bool, - value_s TEXT, - value_ref TEXT, - created_at INT -) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - for $row in get_metadata_core($stream_ref, $key, $ref, $limit, $offset, $order_by) { - RETURN NEXT $row.row_id, $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref, $row.created_at; - } -}; - --- Compatibility wrapper that returns single value from get_metadata_core (for functions expecting single value) -CREATE OR REPLACE ACTION get_metadata_priv_single( - $stream_ref INT, - $key TEXT, - $ref TEXT -) PRIVATE view returns (value TEXT) { - $result TEXT := NULL; - for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { - $result := $row.value_ref; - } - RETURN $result; -}; - -/** - * get_latest_metadata: Retrieves the latest metadata for a stream. - */ -CREATE OR REPLACE ACTION get_latest_metadata_core( - $stream_ref INT, - $key TEXT, - $ref TEXT -) PRIVATE view returns table( - value_i INT, - value_f NUMERIC(36,18), - value_b BOOL, - value_s TEXT, - value_ref TEXT -) { - for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { - RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; - } -}; - -CREATE OR REPLACE ACTION get_latest_metadata( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT, - $ref TEXT -) PUBLIC view returns table( - value_i INT, - value_f NUMERIC(36,18), - value_b BOOL, - value_s TEXT, - value_ref TEXT -) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - - for $row in get_latest_metadata_core($stream_ref, $key, $ref) { - RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; - } -}; - -/** - * get_latest_metadata_int: Retrieves the latest metadata value for a stream. - */ -CREATE OR REPLACE ACTION get_latest_metadata_int_core( - $stream_ref INT, - $key TEXT -) PRIVATE view returns (value INT) { - $result INT := NULL; - for $row in get_latest_metadata_core($stream_ref, $key, NULL) { - $result := $row.value_i; - } - RETURN $result; -}; - -CREATE OR REPLACE ACTION get_latest_metadata_int( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT -) PUBLIC view returns (value INT) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - return get_latest_metadata_int_core($stream_ref, $key); -}; - -/** - * get_latest_metadata_ref: Retrieves the latest metadata value for a stream. - */ -CREATE OR REPLACE ACTION get_latest_metadata_ref_core( - $stream_ref INT, - $key TEXT, - $ref TEXT -) PRIVATE view returns (value TEXT) { - $result TEXT := NULL; - for $row in get_latest_metadata_core($stream_ref, $key, $ref) { - $result := $row.value_ref; - } - RETURN $result; -}; - -CREATE OR REPLACE ACTION get_latest_metadata_ref( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT, - $ref TEXT -) PUBLIC view returns (value TEXT) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - return get_latest_metadata_ref_core($stream_ref, $key, $ref); -}; - -/** - * get_latest_metadata_bool: Retrieves the latest metadata value for a stream. - */ -CREATE OR REPLACE ACTION get_latest_metadata_bool_core( - $stream_ref INT, - $key TEXT -) PRIVATE view returns (value BOOL) { - $result BOOL := NULL; - for $row in get_latest_metadata_core($stream_ref, $key, NULL) { - $result := $row.value_b; - } - RETURN $result; -}; - -CREATE OR REPLACE ACTION get_latest_metadata_bool( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT -) PUBLIC view returns (value BOOL) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - return get_latest_metadata_bool_core($stream_ref, $key); -}; - -/** - * get_latest_metadata_string: Retrieves the latest metadata value for a stream. - */ -CREATE OR REPLACE ACTION get_latest_metadata_string_core( - $stream_ref INT, - $key TEXT -) PRIVATE view returns (value TEXT) { - $result TEXT := NULL; - for $row in get_latest_metadata_core($stream_ref, $key, NULL) { - $result := $row.value_s; - } - RETURN $result; -}; - -CREATE OR REPLACE ACTION get_latest_metadata_string( - $data_provider TEXT, - $stream_id TEXT, - $key TEXT -) PUBLIC view returns (value TEXT) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - return get_latest_metadata_string_core($stream_ref, $key); -}; - -/** - * get_category_streams: Retrieves all streams in a category (composed stream). - * For primitive streams, returns just the stream itself. - * For composed streams, recursively traverses taxonomy to find all substreams. - * It doesn't check for the existence of the substreams, it just returns them. - */ -CREATE OR REPLACE ACTION get_category_streams( - $data_provider TEXT, - $stream_id TEXT, - $active_from INT, - $active_to INT -) PUBLIC view returns table(data_provider TEXT, stream_id TEXT) { - $data_provider := LOWER($data_provider); - - -- Check if stream exists (get_stream_id returns NULL if stream doesn't exist) - $stream_ref := get_stream_id($data_provider, $stream_id); - IF $stream_ref IS NULL { - ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); - } - - -- Always return itself first - RETURN NEXT $data_provider, $stream_id; - - -- For primitive streams, just return the stream itself - if is_primitive_stream($data_provider, $stream_id) == true { - RETURN; - } - - -- Set boundaries for time intervals - $max_int8 INT := 9223372036854775000; - $effective_active_from INT := COALESCE($active_from, 0); - $effective_active_to INT := COALESCE($active_to, $max_int8); - - -- Get all substreams with proper recursive traversal - return WITH RECURSIVE substreams AS ( - /*------------------------------------------------------------------ - * (1) Base Case: overshadow logic for ($data_provider, $stream_id). - * - For each distinct start_time, pick the row with the max group_sequence. - * - next_start is used to define [group_sequence_start, group_sequence_end]. - *------------------------------------------------------------------*/ - SELECT - base.stream_ref, - base.child_stream_ref, - - -- The interval during which this row is active: - base.start_time AS group_sequence_start, - COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end - FROM ( - -- Find rows with maximum group_sequence for each start_time - SELECT - t.stream_ref, - t.child_stream_ref, - t.start_time, - t.group_sequence, - MAX(t.group_sequence) OVER ( - PARTITION BY t.stream_ref, t.start_time - ) AS max_group_sequence - FROM taxonomies t - WHERE t.stream_ref = $stream_ref - AND t.disabled_at IS NULL - AND t.start_time <= $effective_active_to - AND t.start_time >= COALESCE(( - -- Find the most recent taxonomy at or before effective_active_from - SELECT t2.start_time - FROM taxonomies t2 - WHERE t2.stream_ref = t.stream_ref - AND t2.disabled_at IS NULL - AND t2.start_time <= $effective_active_from - ORDER BY t2.start_time DESC, t2.group_sequence DESC - LIMIT 1 - ), 0 - ) - ) base - JOIN ( - /* Distinct start_times for top-level (dp, sid), used for LEAD() */ - SELECT - dt.stream_ref, - dt.start_time, - LEAD(dt.start_time) OVER ( - PARTITION BY dt.stream_ref - ORDER BY dt.start_time - ) AS next_start - FROM ( - SELECT DISTINCT - t.stream_ref, - t.start_time - FROM taxonomies t - WHERE t.stream_ref = $stream_ref - AND t.disabled_at IS NULL - AND t.start_time <= $effective_active_to - AND t.start_time >= COALESCE(( - SELECT t2.start_time - FROM taxonomies t2 - WHERE t2.stream_ref = t.stream_ref - AND t2.disabled_at IS NULL - AND t2.start_time <= $effective_active_from - ORDER BY t2.start_time DESC, t2.group_sequence DESC - LIMIT 1 - ), 0 - ) - ) dt - ) ot - ON base.stream_ref = ot.stream_ref - AND base.start_time = ot.start_time - WHERE base.group_sequence = base.max_group_sequence - - UNION - - /*------------------------------------------------------------------ - * (2) Recursive Child-Level Overshadow: - * For each discovered child, gather overshadow rows for that child - * and produce intervals that overlap the parent's own active interval. - *------------------------------------------------------------------*/ - SELECT - parent.stream_ref, - child.child_stream_ref, - - -- Intersection of parent's active interval and child's: - GREATEST(parent.group_sequence_start, child.start_time) AS group_sequence_start, - LEAST(parent.group_sequence_end, child.group_sequence_end) AS group_sequence_end - FROM substreams parent - JOIN ( - /* Child overshadow logic, same pattern as above but for child dp/sid. */ - SELECT - base.stream_ref, - base.child_stream_ref, - base.start_time, - COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end - FROM ( - SELECT - t.stream_ref, - t.child_stream_ref, - t.start_time, - t.group_sequence, - MAX(t.group_sequence) OVER ( - PARTITION BY t.stream_ref, t.start_time - ) AS max_group_sequence - FROM taxonomies t - WHERE t.disabled_at IS NULL - AND t.start_time <= $effective_active_to - AND t.start_time >= COALESCE(( - -- Most recent taxonomy at or before effective_from - SELECT t2.start_time - FROM taxonomies t2 - WHERE t2.stream_ref = t.stream_ref - AND t2.disabled_at IS NULL - AND t2.start_time <= $effective_active_from - ORDER BY t2.start_time DESC, t2.group_sequence DESC - LIMIT 1 - ), 0 - ) - ) base - JOIN ( - /* Distinct start_times at child level */ - SELECT - dt.stream_ref, - dt.start_time, - LEAD(dt.start_time) OVER ( - PARTITION BY dt.stream_ref - ORDER BY dt.start_time - ) AS next_start - FROM ( - SELECT DISTINCT - t.stream_ref, - t.start_time - FROM taxonomies t - WHERE t.disabled_at IS NULL - AND t.start_time <= $effective_active_to - AND t.start_time >= COALESCE(( - SELECT t2.start_time - FROM taxonomies t2 - WHERE t2.stream_ref = t.stream_ref - AND t2.disabled_at IS NULL - AND t2.start_time <= $effective_active_from - ORDER BY t2.start_time DESC, t2.group_sequence DESC - LIMIT 1 - ), 0 - ) - ) dt - ) ot - ON base.stream_ref = ot.stream_ref - AND base.start_time = ot.start_time - WHERE base.group_sequence = base.max_group_sequence - ) child - ON child.stream_ref = parent.child_stream_ref - - /* Overlap check: child's interval must intersect parent's */ - WHERE child.start_time <= parent.group_sequence_end - AND child.group_sequence_end >= parent.group_sequence_start - ) - SELECT DISTINCT - child_dp.address as child_data_provider, - child_s.stream_id as child_stream_id - FROM substreams sub - JOIN streams child_s ON sub.child_stream_ref = child_s.id - JOIN data_providers child_dp ON child_s.data_provider_id = child_dp.id; -}; - -/** - * stream_exists: Simple check if a stream exists in the database. - */ -CREATE OR REPLACE ACTION stream_exists( - $data_provider TEXT, - $stream_id TEXT -) PUBLIC view returns (result BOOL) { - $data_provider := LOWER($data_provider); - $stream_ref := get_stream_id($data_provider, $stream_id); - - for $row in SELECT 1 FROM streams WHERE id = $stream_ref { - return true; - } - return false; -}; - -/** - * stream_exists_batch_core: Private version that uses stream refs directly. - * Checks if multiple streams exist using their stream references. - * Returns false if any stream refs are null (indicating non-existent streams). - * Returns true only if all streams exist and no stream refs are null. - */ -CREATE OR REPLACE ACTION stream_exists_batch_core( - $stream_refs INT[] -) PRIVATE VIEW RETURNS (result BOOL) { - -- Use UNNEST for efficient batch processing - for $row in SELECT CASE - WHEN EXISTS ( - SELECT 1 FROM UNNEST($stream_refs) AS t(stream_ref) WHERE t.stream_ref IS NULL - ) THEN false - ELSE NOT EXISTS ( - SELECT 1 - FROM UNNEST($stream_refs) AS t(stream_ref) - LEFT JOIN streams s ON s.id = t.stream_ref - WHERE s.id IS NULL - AND t.stream_ref IS NOT NULL - ) - END AS result - FROM (SELECT 1) dummy { - return $row.result; - } - return false; -}; - -/** - * is_primitive_stream_batch_core: Private version that uses stream refs directly. - * Checks if multiple streams are primitive using their stream references. - * Returns false if any stream refs are null (indicating non-existent streams). - * Returns true only if all streams exist and are primitive. - */ -CREATE OR REPLACE ACTION is_primitive_stream_batch_core( - $stream_refs INT[] -) PRIVATE VIEW RETURNS (result BOOL) { - -- Use UNNEST for optimal performance - direct array processing without recursion - for $row in SELECT CASE - WHEN EXISTS ( - SELECT 1 - FROM UNNEST($stream_refs) AS t(stream_ref) - WHERE t.stream_ref IS NULL - ) THEN false - ELSE NOT EXISTS ( - SELECT 1 - FROM UNNEST($stream_refs) AS t(stream_ref) - JOIN streams s ON s.id = t.stream_ref - WHERE s.stream_type != 'primitive' - AND t.stream_ref IS NOT NULL - ) - END AS result - FROM (SELECT 1) dummy { - return $row.result; - } - return false; -}; - -/** - * stream_exists_batch: Checks existence of multiple streams in a single query. - * Returns a table with existence status for each stream. - */ -CREATE OR REPLACE ACTION stream_exists_batch( - $data_providers TEXT[], - $stream_ids TEXT[] -) PUBLIC view returns table( - data_provider TEXT, - stream_id TEXT, - stream_exists BOOL -) { - -- Lowercase data providers directly using UNNEST for efficiency - - -- Check that arrays have the same length - if array_length($data_providers) != array_length($stream_ids) { - ERROR('Data providers and stream IDs arrays must have the same length'); - } - - -- Use UNNEST for optimal performance with direct LOWER operations - RETURN SELECT - t.data_provider, - t.stream_id, - CASE WHEN s.data_provider IS NOT NULL THEN true ELSE false END AS stream_exists - FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) - LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) - LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; -}; - -CREATE OR REPLACE ACTION transfer_stream_ownership( - $data_provider TEXT, - $stream_id TEXT, - $new_owner TEXT -) PUBLIC { - $data_provider := LOWER($data_provider); - $new_owner := LOWER($new_owner); - $lower_caller := LOWER(@caller); - $stream_ref := get_stream_id($data_provider, $stream_id); - - if !is_stream_owner($data_provider, $stream_id, $lower_caller) { - ERROR('Only stream owner can transfer ownership'); - } - - -- Check if new owner is a valid ethereum address - if NOT check_ethereum_address($new_owner) { - ERROR('Invalid new owner address. Must be a valid Ethereum address: ' || $new_owner); - } - - -- Update the stream_owner metadata - UPDATE metadata SET value_ref = LOWER($new_owner) - WHERE metadata_key = 'stream_owner' - AND stream_ref = $stream_ref; -}; - -/** - * filter_streams_by_existence: Filters streams based on existence. - * Can return either existing or non-existing streams based on existing_only flag. - * Takes arrays of data providers and stream IDs as input. - * Uses efficient WITH RECURSIVE pattern for batch processing. - */ -CREATE OR REPLACE ACTION filter_streams_by_existence( - $data_providers TEXT[], - $stream_ids TEXT[], - $existing_only BOOL -) PUBLIC view returns table( - data_provider TEXT, - stream_id TEXT -) { - -- Lowercase data providers directly using UNNEST for efficiency - - -- default to return existing streams - if $existing_only IS NULL { - $existing_only := true; - } - -- Check that arrays have the same length - if array_length($data_providers) != array_length($stream_ids) { - ERROR('Data providers and stream IDs arrays must have the same length'); + -- Ensure non-negative values for PostgreSQL compatibility + if $limit < 0 { + $limit := 0; } - - -- Use UNNEST for efficient batch processing - RETURN SELECT - t.data_provider, - t.stream_id - FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) - LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) - LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id - WHERE (CASE WHEN s.id IS NOT NULL THEN true ELSE false END) = $existing_only; -}; - -CREATE OR REPLACE ACTION list_streams( - $data_provider TEXT, - $limit INT, - $offset INT, - $order_by TEXT, - $block_height INT -) PUBLIC view returns table( - data_provider TEXT, - stream_id TEXT, - stream_type TEXT, - created_at INT8 -) { - $data_provider := LOWER($data_provider); - - if $limit > 5000 { - ERROR('Limit exceeds maximum allowed value of 5000'); - } - if $limit IS NULL { - $limit := 5000; - } - if $limit == 0 { - $limit := 5000; - } - if $offset IS NULL { + if $offset < 0 { $offset := 0; } - if $order_by IS NULL { - $order_by := 'created_at DESC'; - } - if $order_by == '' { - $order_by := 'created_at DESC'; + + -- Get current block height for default behavior + $current_block INT8 := @height; + + -- Determine effective height range, if none given, get all metadata + $effective_from INT8 := COALESCE($from_height, 0); + $effective_to INT8 := COALESCE($to_height, $current_block); + + -- Validate height range + if $effective_from > $effective_to { + ERROR('Invalid height range: from_height (' || $effective_from::TEXT || ') > to_height (' || $effective_to::TEXT || ')'); } - RETURN SELECT - dp.address as data_provider, - s.stream_id, - s.stream_type, - s.created_at - FROM streams s - JOIN data_providers dp ON s.data_provider_id = dp.id - -- do not use LOWER on dp.address, or it will break the index lookup - WHERE ($data_provider IS NULL OR $data_provider = '' OR dp.address = LOWER($data_provider)) - AND s.created_at > $block_height - ORDER BY - CASE WHEN $order_by = 'created_at DESC' THEN s.created_at END DESC, - CASE WHEN $order_by = 'created_at ASC' THEN s.created_at END ASC, - CASE WHEN $order_by = 'stream_id ASC' THEN stream_id END ASC, - CASE WHEN $order_by = 'stream_id DESC' THEN stream_id END DESC, - CASE WHEN $order_by = 'stream_type ASC' THEN stream_type END ASC, - CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC - LIMIT $limit OFFSET $offset; + RETURN SELECT + stream_ref, + row_id, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at + FROM metadata + WHERE metadata_key = $key + AND disabled_at IS NULL + -- do not use LOWER on value_ref, or it will break the index lookup + AND ($ref IS NULL OR value_ref = LOWER($ref)) + AND created_at >= $effective_from + AND created_at <= $effective_to + ORDER BY created_at ASC + LIMIT $limit OFFSET $offset; }; \ No newline at end of file diff --git a/tests/streams/utils/procedure/execute.go b/tests/streams/utils/procedure/execute.go index b39994569..455778c43 100644 --- a/tests/streams/utils/procedure/execute.go +++ b/tests/streams/utils/procedure/execute.go @@ -843,3 +843,46 @@ func GetTaxonomiesForStreams(ctx context.Context, input GetTaxonomiesForStreamsI return processResultRows(resultRows) } + +// ListMetadataByHeight executes list_metadata_by_height action +func ListMetadataByHeight(ctx context.Context, input ListMetadataByHeightInput) ([]ResultRow, error) { + deployer, err := util.NewEthereumAddressFromBytes(input.Platform.Deployer) + if err != nil { + return nil, errors.Wrap(err, "error in ListMetadataByHeight.NewEthereumAddressFromBytes") + } + + txContext := &common.TxContext{ + Ctx: ctx, + BlockContext: &common.BlockContext{Height: input.Height}, + Signer: input.Platform.Deployer, + Caller: deployer.Address(), + TxID: input.Platform.Txid(), + } + + engineContext := &common.EngineContext{ + TxContext: txContext, + } + + var resultRows [][]any + r, err := input.Platform.Engine.Call(engineContext, input.Platform.DB, "", "list_metadata_by_height", []any{ + input.Key, + input.Value, + input.FromHeight, + input.ToHeight, + input.Limit, + input.Offset, + }, func(row *common.Row) error { + values := make([]any, len(row.Values)) + copy(values, row.Values) + resultRows = append(resultRows, values) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "error in ListMetadataByHeight.Call") + } + if r.Error != nil { + return nil, errors.Wrap(r.Error, "error in ListMetadataByHeight.Call") + } + + return processResultRows(resultRows) +} diff --git a/tests/streams/utils/procedure/types.go b/tests/streams/utils/procedure/types.go index 5b7b02cc5..19b8322bf 100644 --- a/tests/streams/utils/procedure/types.go +++ b/tests/streams/utils/procedure/types.go @@ -159,3 +159,14 @@ type GetTaxonomiesForStreamsInput struct { LatestOnly *bool Height int64 } + +type ListMetadataByHeightInput struct { + Platform *kwilTesting.Platform + Key string + Value string + FromHeight *int64 + ToHeight *int64 + Limit *int + Offset *int + Height int64 +} From 12ebc94b8a93b36a79e296ccceb1c212573229e5 Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 22:02:13 +0700 Subject: [PATCH 2/7] chore: uncomment actions --- internal/migrations/001-common-actions.sql | 2606 ++++++++++---------- 1 file changed, 1303 insertions(+), 1303 deletions(-) diff --git a/internal/migrations/001-common-actions.sql b/internal/migrations/001-common-actions.sql index b4f7ec2ae..d331c9db3 100644 --- a/internal/migrations/001-common-actions.sql +++ b/internal/migrations/001-common-actions.sql @@ -1,1325 +1,1325 @@ --- /** --- * create_data_provider: Register new data provider --- */ --- CREATE OR REPLACE ACTION create_data_provider( --- $address TEXT --- ) PUBLIC { --- $lower_caller TEXT := LOWER(@caller); --- -- Permission Check: Ensure caller has the 'system:network_writer' role. --- $has_permission BOOL := false; --- for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { --- if $row.wallet = $lower_caller AND $row.is_member { --- $has_permission := true; --- break; --- } --- } --- if NOT $has_permission { --- ERROR('Caller does not have the required system:network_writer role to create data provider.'); --- } - --- $lower_address TEXT := LOWER($address); - --- -- Check if address provided is a valid ethereum address --- if NOT check_ethereum_address($lower_address) { --- ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $lower_address); --- } - --- INSERT INTO data_providers (id, address, created_at) --- SELECT --- COALESCE(MAX(id), 0) + 1, --- $lower_address, --- @height --- FROM data_providers --- ON CONFLICT DO NOTHING; --- }; - --- /** --- * create_stream: Creates a new stream with required metadata. --- * Validates stream_id format, data provider address, and stream type. --- * Sets default metadata including type, owner, visibility, and readonly keys. --- */ --- CREATE OR REPLACE ACTION create_stream( --- $stream_id TEXT, --- $stream_type TEXT --- ) PUBLIC { --- -- Delegate to batch implementation for single-stream consistency. --- create_streams( ARRAY[$stream_id], ARRAY[$stream_type] ); --- }; - --- /** --- * create_streams: Creates multiple streams at once. --- * Validates stream_id format, data provider address, and stream type. --- * Sets default metadata including type, owner, visibility, and readonly keys. --- */ --- CREATE OR REPLACE ACTION create_streams( --- $stream_ids TEXT[], --- $stream_types TEXT[] --- ) PUBLIC { --- $lower_caller TEXT := LOWER(@caller); --- -- Permission Check: Ensure caller has the 'system:network_writer' role. --- $has_permission BOOL := false; --- for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { --- if $row.wallet = $lower_caller AND $row.is_member { --- $has_permission := true; --- break; --- } --- } --- if NOT $has_permission { --- ERROR('Caller does not have the required system:network_writer role to create streams.'); --- } - --- -- Get caller's address (data provider) first --- $data_provider TEXT := $lower_caller; - --- -- Check if caller is a valid ethereum address --- if NOT check_ethereum_address($data_provider) { --- ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $data_provider); --- } - --- -- Check if stream_ids and stream_types arrays have the same length --- if array_length($stream_ids) != array_length($stream_types) { --- ERROR('Stream IDs and stream types arrays must have the same length'); --- } - --- -- Validate stream IDs --- for $validation_result in validate_stream_ids_format_batch($stream_ids) { --- if NOT $validation_result.is_valid { --- ERROR('Invalid stream_id format: ' || $validation_result.stream_id || ' - ' || --- $validation_result.error_reason); --- } --- } - --- -- Validate stream types using dedicated private function --- for $validation_result in validate_stream_types_batch($stream_types) { --- IF $validation_result.error_reason != '' { --- ERROR('Invalid stream type at position ' || $validation_result.position || ': ' || --- $validation_result.stream_type || ' - ' || $validation_result.error_reason); --- } --- } - --- $base_uuid := uuid_generate_kwil('create_streams_' || @txid); - --- -- Get the data provider id --- $data_provider_id INT; --- $dp_found BOOL := false; --- for $data_provider_row in SELECT id --- FROM data_providers --- WHERE address = $data_provider --- LIMIT 1 { --- $dp_found := true; --- $data_provider_id := $data_provider_row.id; --- } - --- if $dp_found = false { --- ERROR('Data provider not found: ' || $data_provider); --- } +/** + * create_data_provider: Register new data provider + */ +CREATE OR REPLACE ACTION create_data_provider( + $address TEXT +) PUBLIC { + $lower_caller TEXT := LOWER(@caller); + -- Permission Check: Ensure caller has the 'system:network_writer' role. + $has_permission BOOL := false; + for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { + if $row.wallet = $lower_caller AND $row.is_member { + $has_permission := true; + break; + } + } + if NOT $has_permission { + ERROR('Caller does not have the required system:network_writer role to create data provider.'); + } + + $lower_address TEXT := LOWER($address); + + -- Check if address provided is a valid ethereum address + if NOT check_ethereum_address($lower_address) { + ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $lower_address); + } + + INSERT INTO data_providers (id, address, created_at) + SELECT + COALESCE(MAX(id), 0) + 1, + $lower_address, + @height + FROM data_providers + ON CONFLICT DO NOTHING; +}; + +/** + * create_stream: Creates a new stream with required metadata. + * Validates stream_id format, data provider address, and stream type. + * Sets default metadata including type, owner, visibility, and readonly keys. + */ +CREATE OR REPLACE ACTION create_stream( + $stream_id TEXT, + $stream_type TEXT +) PUBLIC { + -- Delegate to batch implementation for single-stream consistency. + create_streams( ARRAY[$stream_id], ARRAY[$stream_type] ); +}; + +/** + * create_streams: Creates multiple streams at once. + * Validates stream_id format, data provider address, and stream type. + * Sets default metadata including type, owner, visibility, and readonly keys. + */ +CREATE OR REPLACE ACTION create_streams( + $stream_ids TEXT[], + $stream_types TEXT[] +) PUBLIC { + $lower_caller TEXT := LOWER(@caller); + -- Permission Check: Ensure caller has the 'system:network_writer' role. + $has_permission BOOL := false; + for $row in are_members_of('system', 'network_writer', ARRAY[$lower_caller]) { + if $row.wallet = $lower_caller AND $row.is_member { + $has_permission := true; + break; + } + } + if NOT $has_permission { + ERROR('Caller does not have the required system:network_writer role to create streams.'); + } + + -- Get caller's address (data provider) first + $data_provider TEXT := $lower_caller; + + -- Check if caller is a valid ethereum address + if NOT check_ethereum_address($data_provider) { + ERROR('Invalid data provider address. Must be a valid Ethereum address: ' || $data_provider); + } + + -- Check if stream_ids and stream_types arrays have the same length + if array_length($stream_ids) != array_length($stream_types) { + ERROR('Stream IDs and stream types arrays must have the same length'); + } + + -- Validate stream IDs + for $validation_result in validate_stream_ids_format_batch($stream_ids) { + if NOT $validation_result.is_valid { + ERROR('Invalid stream_id format: ' || $validation_result.stream_id || ' - ' || + $validation_result.error_reason); + } + } + + -- Validate stream types using dedicated private function + for $validation_result in validate_stream_types_batch($stream_types) { + IF $validation_result.error_reason != '' { + ERROR('Invalid stream type at position ' || $validation_result.position || ': ' || + $validation_result.stream_type || ' - ' || $validation_result.error_reason); + } + } + + $base_uuid := uuid_generate_kwil('create_streams_' || @txid); + + -- Get the data provider id + $data_provider_id INT; + $dp_found BOOL := false; + for $data_provider_row in SELECT id + FROM data_providers + WHERE address = $data_provider + LIMIT 1 { + $dp_found := true; + $data_provider_id := $data_provider_row.id; + } + + if $dp_found = false { + ERROR('Data provider not found: ' || $data_provider); + } --- -- Create the streams using UNNEST for optimal performance --- INSERT INTO streams (id, data_provider_id, data_provider, stream_id, stream_type, created_at) --- SELECT --- ROW_NUMBER() OVER (ORDER BY t.stream_id) + COALESCE((SELECT MAX(id) FROM streams), 0) AS id, --- $data_provider_id, --- $data_provider, --- t.stream_id, --- t.stream_type, --- @height --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type); + -- Create the streams using UNNEST for optimal performance + INSERT INTO streams (id, data_provider_id, data_provider, stream_id, stream_type, created_at) + SELECT + ROW_NUMBER() OVER (ORDER BY t.stream_id) + COALESCE((SELECT MAX(id) FROM streams), 0) AS id, + $data_provider_id, + $data_provider, + t.stream_id, + t.stream_type, + @height + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type); --- -- Create metadata for the streams using UNNEST for optimal performance --- -- Insert stream_owner metadata --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at, --- disabled_at, --- stream_ref --- ) --- SELECT --- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'stream_owner' || '1')::UUID, --- 'stream_owner'::TEXT, --- NULL::INT, --- NULL::NUMERIC(36,18), --- NULL::BOOL, --- NULL::TEXT, --- LOWER($data_provider)::TEXT, --- @height, --- NULL::INT, --- s.id --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) --- JOIN data_providers dp ON dp.address = $data_provider --- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - --- -- Insert read_visibility metadata --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at, --- disabled_at, --- stream_ref --- ) --- SELECT --- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'read_visibility' || '2')::UUID, --- 'read_visibility'::TEXT, --- 0::INT, --- NULL::NUMERIC(36,18), --- NULL::BOOL, --- NULL::TEXT, --- NULL::TEXT, --- @height, --- NULL::INT, --- s.id --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) --- JOIN data_providers dp ON dp.address = $data_provider --- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - --- -- Insert readonly_key metadata (stream_owner) --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at, --- disabled_at, --- stream_ref --- ) --- SELECT --- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '3')::UUID, --- 'readonly_key'::TEXT, --- NULL::INT, --- NULL::NUMERIC(36,18), --- NULL::BOOL, --- 'stream_owner'::TEXT, --- NULL::TEXT, --- @height, --- NULL::INT, --- s.id --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) --- JOIN data_providers dp ON dp.address = $data_provider --- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - --- -- Insert readonly_key metadata (readonly_key) --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at, --- disabled_at, --- stream_ref --- ) --- SELECT --- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '4')::UUID, --- 'readonly_key'::TEXT, --- NULL::INT, --- NULL::NUMERIC(36,18), --- NULL::BOOL, --- 'readonly_key'::TEXT, --- NULL::TEXT, --- @height, --- NULL::INT, --- s.id --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) --- JOIN data_providers dp ON dp.address = $data_provider --- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - --- -- Insert type metadata --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at, --- disabled_at, --- stream_ref --- ) --- SELECT --- uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'type' || '5')::UUID, --- 'type'::TEXT, --- NULL::INT, --- NULL::NUMERIC(36,18), --- NULL::BOOL, --- t.stream_type, --- NULL::TEXT, --- @height, --- NULL::INT, --- s.id --- FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) --- JOIN data_providers dp ON dp.address = $data_provider --- JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; - --- }; - - --- /** --- * insert_metadata: Adds metadata to a stream. --- * Validates caller is stream owner and handles different value types. --- * Prevents modification of readonly keys. --- */ --- CREATE OR REPLACE ACTION insert_metadata( --- -- not necessarily the caller is the original deployer of the stream --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT, --- $value TEXT, --- $val_type TEXT --- ) PUBLIC { --- -- Initialize value variables --- $value_i INT; --- $value_s TEXT; --- $value_f DECIMAL(36,18); --- $value_b BOOL; --- $value_ref TEXT; --- $data_provider := LOWER($data_provider); --- $lower_caller := LOWER(@caller); + -- Create metadata for the streams using UNNEST for optimal performance + -- Insert stream_owner metadata + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at, + disabled_at, + stream_ref + ) + SELECT + uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'stream_owner' || '1')::UUID, + 'stream_owner'::TEXT, + NULL::INT, + NULL::NUMERIC(36,18), + NULL::BOOL, + NULL::TEXT, + LOWER($data_provider)::TEXT, + @height, + NULL::INT, + s.id + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) + JOIN data_providers dp ON dp.address = $data_provider + JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + + -- Insert read_visibility metadata + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at, + disabled_at, + stream_ref + ) + SELECT + uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'read_visibility' || '2')::UUID, + 'read_visibility'::TEXT, + 0::INT, + NULL::NUMERIC(36,18), + NULL::BOOL, + NULL::TEXT, + NULL::TEXT, + @height, + NULL::INT, + s.id + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) + JOIN data_providers dp ON dp.address = $data_provider + JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + + -- Insert readonly_key metadata (stream_owner) + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at, + disabled_at, + stream_ref + ) + SELECT + uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '3')::UUID, + 'readonly_key'::TEXT, + NULL::INT, + NULL::NUMERIC(36,18), + NULL::BOOL, + 'stream_owner'::TEXT, + NULL::TEXT, + @height, + NULL::INT, + s.id + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) + JOIN data_providers dp ON dp.address = $data_provider + JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + + -- Insert readonly_key metadata (readonly_key) + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at, + disabled_at, + stream_ref + ) + SELECT + uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'readonly_key' || '4')::UUID, + 'readonly_key'::TEXT, + NULL::INT, + NULL::NUMERIC(36,18), + NULL::BOOL, + 'readonly_key'::TEXT, + NULL::TEXT, + @height, + NULL::INT, + s.id + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) + JOIN data_providers dp ON dp.address = $data_provider + JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + + -- Insert type metadata + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at, + disabled_at, + stream_ref + ) + SELECT + uuid_generate_v5($base_uuid, 'metadata' || $data_provider || t.stream_id || 'type' || '5')::UUID, + 'type'::TEXT, + NULL::INT, + NULL::NUMERIC(36,18), + NULL::BOOL, + t.stream_type, + NULL::TEXT, + @height, + NULL::INT, + s.id + FROM UNNEST($stream_ids, $stream_types) AS t(stream_id, stream_type) + JOIN data_providers dp ON dp.address = $data_provider + JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; + +}; + + +/** + * insert_metadata: Adds metadata to a stream. + * Validates caller is stream owner and handles different value types. + * Prevents modification of readonly keys. + */ +CREATE OR REPLACE ACTION insert_metadata( + -- not necessarily the caller is the original deployer of the stream + $data_provider TEXT, + $stream_id TEXT, + $key TEXT, + $value TEXT, + $val_type TEXT +) PUBLIC { + -- Initialize value variables + $value_i INT; + $value_s TEXT; + $value_f DECIMAL(36,18); + $value_b BOOL; + $value_ref TEXT; + $data_provider := LOWER($data_provider); + $lower_caller := LOWER(@caller); --- -- Check if caller is the stream owner --- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { --- ERROR('Only stream owner can insert metadata'); --- } + -- Check if caller is the stream owner + if !is_stream_owner($data_provider, $stream_id, $lower_caller) { + ERROR('Only stream owner can insert metadata'); + } --- -- Set the appropriate value based on type --- if $val_type = 'int' { --- $value_i := $value::INT; --- } elseif $val_type = 'string' { --- $value_s := $value; --- } elseif $val_type = 'bool' { --- $value_b := $value::BOOL; --- } elseif $val_type = 'ref' { --- $value_ref := $value; --- } elseif $val_type = 'float' { --- $value_f := $value::DECIMAL(36,18); --- } else { --- ERROR(FORMAT('Unknown type used "%s". Valid types = "float" | "bool" | "int" | "ref" | "string"', $val_type)); --- } - --- $stream_ref := get_stream_id($data_provider, $stream_id); + -- Set the appropriate value based on type + if $val_type = 'int' { + $value_i := $value::INT; + } elseif $val_type = 'string' { + $value_s := $value; + } elseif $val_type = 'bool' { + $value_b := $value::BOOL; + } elseif $val_type = 'ref' { + $value_ref := $value; + } elseif $val_type = 'float' { + $value_f := $value::DECIMAL(36,18); + } else { + ERROR(FORMAT('Unknown type used "%s". Valid types = "float" | "bool" | "int" | "ref" | "string"', $val_type)); + } + + $stream_ref := get_stream_id($data_provider, $stream_id); --- -- Check if the key is read-only --- $is_readonly BOOL := false; --- for $row in SELECT * FROM metadata --- WHERE stream_ref = $stream_ref --- AND metadata_key = 'readonly_key' --- AND value_s = $key LIMIT 1 { --- $is_readonly := true; --- } + -- Check if the key is read-only + $is_readonly BOOL := false; + for $row in SELECT * FROM metadata + WHERE stream_ref = $stream_ref + AND metadata_key = 'readonly_key' + AND value_s = $key LIMIT 1 { + $is_readonly := true; + } --- if $is_readonly = true { --- ERROR('Cannot insert metadata for read-only key'); --- } + if $is_readonly = true { + ERROR('Cannot insert metadata for read-only key'); + } --- -- Create deterministic UUID for the metadata record --- $uuid_key TEXT := @txid || $key || $value; --- $uuid UUID := uuid_generate_kwil($uuid_key); --- $current_block INT := @height; --- $stream_ref INT := get_stream_id($data_provider, $stream_id); + -- Create deterministic UUID for the metadata record + $uuid_key TEXT := @txid || $key || $value; + $uuid UUID := uuid_generate_kwil($uuid_key); + $current_block INT := @height; + $stream_ref INT := get_stream_id($data_provider, $stream_id); --- -- Insert the metadata --- INSERT INTO metadata ( --- row_id, --- metadata_key, --- value_i, --- value_f, --- value_s, --- value_b, --- value_ref, --- created_at, --- stream_ref --- ) VALUES ( --- $uuid, --- $key, --- $value_i, --- $value_f, --- $value_s, --- $value_b, --- LOWER($value_ref), --- $current_block, --- $stream_ref --- ); --- }; - --- /** --- * disable_metadata: Marks a metadata record as disabled. --- * Validates caller is stream owner and prevents disabling readonly keys. --- */ --- CREATE OR REPLACE ACTION disable_metadata( --- -- not necessarily the caller is the original deployer of the stream --- $data_provider TEXT, --- $stream_id TEXT, --- $row_id UUID --- ) PUBLIC { --- $data_provider := LOWER($data_provider); --- $lower_caller := LOWER(@caller); --- -- Check if caller is the stream owner --- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { --- ERROR('Only stream owner can disable metadata'); --- } + -- Insert the metadata + INSERT INTO metadata ( + row_id, + metadata_key, + value_i, + value_f, + value_s, + value_b, + value_ref, + created_at, + stream_ref + ) VALUES ( + $uuid, + $key, + $value_i, + $value_f, + $value_s, + $value_b, + LOWER($value_ref), + $current_block, + $stream_ref + ); +}; + +/** + * disable_metadata: Marks a metadata record as disabled. + * Validates caller is stream owner and prevents disabling readonly keys. + */ +CREATE OR REPLACE ACTION disable_metadata( + -- not necessarily the caller is the original deployer of the stream + $data_provider TEXT, + $stream_id TEXT, + $row_id UUID +) PUBLIC { + $data_provider := LOWER($data_provider); + $lower_caller := LOWER(@caller); + -- Check if caller is the stream owner + if !is_stream_owner($data_provider, $stream_id, $lower_caller) { + ERROR('Only stream owner can disable metadata'); + } --- $current_block INT := @height; --- $found BOOL := false; --- $metadata_key TEXT; --- $stream_ref INT := get_stream_id($data_provider, $stream_id); + $current_block INT := @height; + $found BOOL := false; + $metadata_key TEXT; + $stream_ref INT := get_stream_id($data_provider, $stream_id); --- -- Get the metadata key first to avoid nested queries --- for $metadata_row in SELECT metadata_key --- FROM metadata --- WHERE row_id = $row_id --- AND stream_ref = $stream_ref --- AND disabled_at IS NULL --- LIMIT 1 { --- $found := true; --- $metadata_key := $metadata_row.metadata_key; --- } + -- Get the metadata key first to avoid nested queries + for $metadata_row in SELECT metadata_key + FROM metadata + WHERE row_id = $row_id + AND stream_ref = $stream_ref + AND disabled_at IS NULL + LIMIT 1 { + $found := true; + $metadata_key := $metadata_row.metadata_key; + } --- if $found = false { --- ERROR('Metadata record not found'); --- } + if $found = false { + ERROR('Metadata record not found'); + } --- -- In a separate step, check if the key is read-only --- $is_readonly BOOL := false; --- for $readonly_row in SELECT * FROM metadata --- WHERE stream_ref = $stream_ref --- AND metadata_key = 'readonly_key' --- AND value_s = $metadata_key LIMIT 1 { --- $is_readonly := true; --- } + -- In a separate step, check if the key is read-only + $is_readonly BOOL := false; + for $readonly_row in SELECT * FROM metadata + WHERE stream_ref = $stream_ref + AND metadata_key = 'readonly_key' + AND value_s = $metadata_key LIMIT 1 { + $is_readonly := true; + } --- if $is_readonly = true { --- ERROR('Cannot disable read-only metadata'); --- } + if $is_readonly = true { + ERROR('Cannot disable read-only metadata'); + } --- -- Update the metadata to mark it as disabled --- UPDATE metadata SET disabled_at = $current_block --- WHERE row_id = $row_id --- AND stream_ref = $stream_ref; --- }; - --- /** --- * check_stream_id_format: Validates stream ID format (st + 30 alphanumeric chars). --- */ --- CREATE OR REPLACE ACTION check_stream_id_format( --- $stream_id TEXT --- ) PUBLIC view returns (result BOOL) { --- -- Check that the stream_id is exactly 32 characters and starts with "st" --- if LENGTH($stream_id) != 32 OR substring($stream_id, 1, 2) != 'st' { --- return false; --- } - --- -- Iterate through each character after the "st" prefix. --- for $i in 3..32 { --- $c TEXT := substring($stream_id, $i, 1); --- if NOT ( --- ($c >= '0' AND $c <= '9') --- OR ($c >= 'a' AND $c <= 'z') --- ) { --- return false; --- } --- } - --- return true; --- }; - --- /** --- * validate_stream_ids_format_batch: Validates multiple stream ID formats efficiently. --- * Returns all stream IDs with their validation status and error details. --- */ --- CREATE OR REPLACE ACTION validate_stream_ids_format_batch( --- $stream_ids TEXT[] --- ) PUBLIC view returns table( --- stream_id TEXT, --- is_valid BOOL, --- error_reason TEXT --- ) { --- -- Pure SQL validation using supported functions only --- RETURN SELECT --- t.stream_id, --- CASE --- WHEN LENGTH(t.stream_id) != 32 THEN false --- WHEN substring(t.stream_id, 1, 2) != 'st' THEN false --- -- Check characters 3-32 are lowercase alphanumeric using basic string functions --- WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN false --- ELSE true --- END AS is_valid, --- CASE --- WHEN LENGTH(t.stream_id) != 32 THEN 'Invalid length (must be 32 characters)' --- WHEN substring(t.stream_id, 1, 2) != 'st' THEN 'Must start with "st"' --- -- Check characters 3-32 are lowercase alphanumeric using basic string functions --- WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN 'Characters 3-32 must be lowercase alphanumeric' --- ELSE '' --- END AS error_reason --- FROM UNNEST($stream_ids) AS t(stream_id); --- }; - --- /** --- * validate_stream_types_batch: Validates multiple stream types efficiently. --- * Returns invalid stream types with their positions and error details. --- */ --- CREATE OR REPLACE ACTION validate_stream_types_batch( --- $stream_types TEXT[] --- ) PRIVATE view returns table( --- position INT, --- stream_type TEXT, --- error_reason TEXT --- ) { --- -- Use CTE with row_number() since WITH ORDINALITY is not supported in Kuneiform --- RETURN WITH indexed_types AS ( --- SELECT --- row_number() OVER () as idx, --- stream_type --- FROM UNNEST($stream_types) AS t(stream_type) --- ) --- SELECT --- idx as position, --- stream_type, --- CASE --- WHEN stream_type NOT IN ('primitive', 'composed') THEN --- 'Stream type must be "primitive" or "composed"' --- ELSE '' --- END AS error_reason --- FROM indexed_types --- WHERE stream_type NOT IN ('primitive', 'composed'); --- }; - --- /** --- * check_ethereum_address: Validates Ethereum address format. --- */ --- CREATE OR REPLACE ACTION check_ethereum_address( --- $data_provider TEXT --- ) PUBLIC view returns (result BOOL) { --- -- Verify the address is exactly 42 characters and starts with "0x" --- if LENGTH($data_provider) != 42 OR substring($data_provider, 1, 2) != '0x' { --- return false; --- } - --- -- Iterate through each character after the "0x" prefix. --- for $i in 3..42 { --- $c TEXT := substring($data_provider, $i, 1); --- if NOT ( --- ($c >= '0' AND $c <= '9') --- OR ($c >= 'a' AND $c <= 'f') --- OR ($c >= 'A' AND $c <= 'F') --- ) { --- return false; --- } --- } - --- return true; --- }; - --- /** --- * delete_stream: Removes a stream and all associated data. --- * Only stream owner can perform this action. --- */ --- CREATE OR REPLACE ACTION delete_stream( --- -- not necessarily the caller is the original deployer of the stream --- $data_provider TEXT, --- $stream_id TEXT --- ) PUBLIC { --- $data_provider := LOWER($data_provider); --- $lower_caller := LOWER(@caller); - --- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { --- ERROR('Only stream owner can delete the stream'); --- } - --- $stream_ref := get_stream_id($data_provider, $stream_id); - --- DELETE FROM streams WHERE id = $stream_ref; --- }; - --- /** --- * is_stream_owner: Checks if caller is the owner of a stream. --- * Uses stream_owner metadata to determine ownership. --- */ --- CREATE OR REPLACE ACTION is_stream_owner_core( --- $stream_ref INT, --- $caller TEXT --- ) PRIVATE view returns (is_owner BOOL) { --- -- Check if the caller is the owner by looking at the latest stream_owner metadata --- for $row in SELECT COALESCE( --- -- do not use LOWER here, or it will break the index lookup --- (SELECT m.value_ref = LOWER($caller) --- FROM metadata m --- WHERE m.stream_ref = $stream_ref --- AND m.metadata_key = 'stream_owner' --- AND m.disabled_at IS NULL --- ORDER BY m.created_at DESC --- LIMIT 1), false --- ) as result { --- return $row.result; --- } --- return false; --- }; - --- CREATE OR REPLACE ACTION is_stream_owner( --- $data_provider TEXT, --- $stream_id TEXT, --- $caller TEXT --- ) PUBLIC view returns (is_owner BOOL) { --- $data_provider := LOWER($data_provider); --- $lower_caller := LOWER($caller); - --- -- Check if the stream exists (get_stream_id returns NULL if stream doesn't exist) --- $stream_ref := get_stream_id($data_provider, $stream_id); --- IF $stream_ref IS NULL { --- ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); --- } --- return is_stream_owner_core($stream_ref, $lower_caller); --- }; - --- /** --- * is_stream_owner_batch: Checks if a wallet is the owner of multiple streams. --- * Processes arrays of data providers and stream IDs efficiently. --- * Returns a table indicating ownership status for each stream. --- */ --- CREATE OR REPLACE ACTION is_stream_owner_batch( --- $data_providers TEXT[], --- $stream_ids TEXT[], --- $wallet TEXT --- ) PUBLIC view returns table( --- data_provider TEXT, --- stream_id TEXT, --- is_owner BOOL --- ) { --- -- Lowercase data providers directly using UNNEST for efficiency - --- -- Check that arrays have the same length --- if array_length($data_providers) != array_length($stream_ids) { --- ERROR('Data providers and stream IDs arrays must have the same length'); --- } - --- -- Check if the wallet is the owner of each stream --- for $row in stream_exists_batch($data_providers, $stream_ids) { --- if !$row.stream_exists { --- ERROR('stream does not exist: data_provider=' || $row.data_provider || ', stream_id=' || $row.stream_id); --- } --- } --- $lowercase_wallet TEXT := LOWER($wallet); - --- -- Use UNNEST for optimal performance with direct LOWER operations --- RETURN SELECT --- t.data_provider, --- t.stream_id, --- CASE WHEN m.value_ref IS NOT NULL AND m.value_ref = $lowercase_wallet THEN true ELSE false END AS is_owner --- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) --- LEFT JOIN ( --- SELECT dp.address as data_provider, s.stream_id, latest.value_ref --- FROM ( --- SELECT md.stream_ref, md.value_ref, --- ROW_NUMBER() OVER (PARTITION BY md.stream_ref ORDER BY md.created_at DESC, md.row_id DESC) AS rn --- FROM metadata md --- WHERE md.metadata_key = 'stream_owner' --- AND md.disabled_at IS NULL --- ) latest --- JOIN streams s ON latest.stream_ref = s.id --- JOIN data_providers dp ON s.data_provider_id = dp.id --- WHERE latest.rn = 1 --- ) m ON LOWER(t.data_provider) = m.data_provider AND t.stream_id = m.stream_id; --- }; - --- /** --- * is_primitive_stream: Determines if a stream is primitive or composed. --- */ --- CREATE OR REPLACE ACTION is_primitive_stream( --- $data_provider TEXT, --- $stream_id TEXT --- ) PUBLIC view returns (is_primitive BOOL) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- for $row in SELECT stream_type FROM streams --- WHERE id = $stream_ref LIMIT 1 { --- return $row.stream_type = 'primitive'; --- } + -- Update the metadata to mark it as disabled + UPDATE metadata SET disabled_at = $current_block + WHERE row_id = $row_id + AND stream_ref = $stream_ref; +}; + +/** + * check_stream_id_format: Validates stream ID format (st + 30 alphanumeric chars). + */ +CREATE OR REPLACE ACTION check_stream_id_format( + $stream_id TEXT +) PUBLIC view returns (result BOOL) { + -- Check that the stream_id is exactly 32 characters and starts with "st" + if LENGTH($stream_id) != 32 OR substring($stream_id, 1, 2) != 'st' { + return false; + } + + -- Iterate through each character after the "st" prefix. + for $i in 3..32 { + $c TEXT := substring($stream_id, $i, 1); + if NOT ( + ($c >= '0' AND $c <= '9') + OR ($c >= 'a' AND $c <= 'z') + ) { + return false; + } + } + + return true; +}; + +/** + * validate_stream_ids_format_batch: Validates multiple stream ID formats efficiently. + * Returns all stream IDs with their validation status and error details. + */ +CREATE OR REPLACE ACTION validate_stream_ids_format_batch( + $stream_ids TEXT[] +) PUBLIC view returns table( + stream_id TEXT, + is_valid BOOL, + error_reason TEXT +) { + -- Pure SQL validation using supported functions only + RETURN SELECT + t.stream_id, + CASE + WHEN LENGTH(t.stream_id) != 32 THEN false + WHEN substring(t.stream_id, 1, 2) != 'st' THEN false + -- Check characters 3-32 are lowercase alphanumeric using basic string functions + WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN false + ELSE true + END AS is_valid, + CASE + WHEN LENGTH(t.stream_id) != 32 THEN 'Invalid length (must be 32 characters)' + WHEN substring(t.stream_id, 1, 2) != 'st' THEN 'Must start with "st"' + -- Check characters 3-32 are lowercase alphanumeric using basic string functions + WHEN LENGTH(trim(lower(substring(t.stream_id, 3, 30)), '0123456789abcdefghijklmnopqrstuvwxyz')) != 0 THEN 'Characters 3-32 must be lowercase alphanumeric' + ELSE '' + END AS error_reason + FROM UNNEST($stream_ids) AS t(stream_id); +}; + +/** + * validate_stream_types_batch: Validates multiple stream types efficiently. + * Returns invalid stream types with their positions and error details. + */ +CREATE OR REPLACE ACTION validate_stream_types_batch( + $stream_types TEXT[] +) PRIVATE view returns table( + position INT, + stream_type TEXT, + error_reason TEXT +) { + -- Use CTE with row_number() since WITH ORDINALITY is not supported in Kuneiform + RETURN WITH indexed_types AS ( + SELECT + row_number() OVER () as idx, + stream_type + FROM UNNEST($stream_types) AS t(stream_type) + ) + SELECT + idx as position, + stream_type, + CASE + WHEN stream_type NOT IN ('primitive', 'composed') THEN + 'Stream type must be "primitive" or "composed"' + ELSE '' + END AS error_reason + FROM indexed_types + WHERE stream_type NOT IN ('primitive', 'composed'); +}; + +/** + * check_ethereum_address: Validates Ethereum address format. + */ +CREATE OR REPLACE ACTION check_ethereum_address( + $data_provider TEXT +) PUBLIC view returns (result BOOL) { + -- Verify the address is exactly 42 characters and starts with "0x" + if LENGTH($data_provider) != 42 OR substring($data_provider, 1, 2) != '0x' { + return false; + } + + -- Iterate through each character after the "0x" prefix. + for $i in 3..42 { + $c TEXT := substring($data_provider, $i, 1); + if NOT ( + ($c >= '0' AND $c <= '9') + OR ($c >= 'a' AND $c <= 'f') + OR ($c >= 'A' AND $c <= 'F') + ) { + return false; + } + } + + return true; +}; + +/** + * delete_stream: Removes a stream and all associated data. + * Only stream owner can perform this action. + */ +CREATE OR REPLACE ACTION delete_stream( + -- not necessarily the caller is the original deployer of the stream + $data_provider TEXT, + $stream_id TEXT +) PUBLIC { + $data_provider := LOWER($data_provider); + $lower_caller := LOWER(@caller); + + if !is_stream_owner($data_provider, $stream_id, $lower_caller) { + ERROR('Only stream owner can delete the stream'); + } + + $stream_ref := get_stream_id($data_provider, $stream_id); + + DELETE FROM streams WHERE id = $stream_ref; +}; + +/** + * is_stream_owner: Checks if caller is the owner of a stream. + * Uses stream_owner metadata to determine ownership. + */ +CREATE OR REPLACE ACTION is_stream_owner_core( + $stream_ref INT, + $caller TEXT +) PRIVATE view returns (is_owner BOOL) { + -- Check if the caller is the owner by looking at the latest stream_owner metadata + for $row in SELECT COALESCE( + -- do not use LOWER here, or it will break the index lookup + (SELECT m.value_ref = LOWER($caller) + FROM metadata m + WHERE m.stream_ref = $stream_ref + AND m.metadata_key = 'stream_owner' + AND m.disabled_at IS NULL + ORDER BY m.created_at DESC + LIMIT 1), false + ) as result { + return $row.result; + } + return false; +}; + +CREATE OR REPLACE ACTION is_stream_owner( + $data_provider TEXT, + $stream_id TEXT, + $caller TEXT +) PUBLIC view returns (is_owner BOOL) { + $data_provider := LOWER($data_provider); + $lower_caller := LOWER($caller); + + -- Check if the stream exists (get_stream_id returns NULL if stream doesn't exist) + $stream_ref := get_stream_id($data_provider, $stream_id); + IF $stream_ref IS NULL { + ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); + } + return is_stream_owner_core($stream_ref, $lower_caller); +}; + +/** + * is_stream_owner_batch: Checks if a wallet is the owner of multiple streams. + * Processes arrays of data providers and stream IDs efficiently. + * Returns a table indicating ownership status for each stream. + */ +CREATE OR REPLACE ACTION is_stream_owner_batch( + $data_providers TEXT[], + $stream_ids TEXT[], + $wallet TEXT +) PUBLIC view returns table( + data_provider TEXT, + stream_id TEXT, + is_owner BOOL +) { + -- Lowercase data providers directly using UNNEST for efficiency + + -- Check that arrays have the same length + if array_length($data_providers) != array_length($stream_ids) { + ERROR('Data providers and stream IDs arrays must have the same length'); + } + + -- Check if the wallet is the owner of each stream + for $row in stream_exists_batch($data_providers, $stream_ids) { + if !$row.stream_exists { + ERROR('stream does not exist: data_provider=' || $row.data_provider || ', stream_id=' || $row.stream_id); + } + } + $lowercase_wallet TEXT := LOWER($wallet); + + -- Use UNNEST for optimal performance with direct LOWER operations + RETURN SELECT + t.data_provider, + t.stream_id, + CASE WHEN m.value_ref IS NOT NULL AND m.value_ref = $lowercase_wallet THEN true ELSE false END AS is_owner + FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) + LEFT JOIN ( + SELECT dp.address as data_provider, s.stream_id, latest.value_ref + FROM ( + SELECT md.stream_ref, md.value_ref, + ROW_NUMBER() OVER (PARTITION BY md.stream_ref ORDER BY md.created_at DESC, md.row_id DESC) AS rn + FROM metadata md + WHERE md.metadata_key = 'stream_owner' + AND md.disabled_at IS NULL + ) latest + JOIN streams s ON latest.stream_ref = s.id + JOIN data_providers dp ON s.data_provider_id = dp.id + WHERE latest.rn = 1 + ) m ON LOWER(t.data_provider) = m.data_provider AND t.stream_id = m.stream_id; +}; + +/** + * is_primitive_stream: Determines if a stream is primitive or composed. + */ +CREATE OR REPLACE ACTION is_primitive_stream( + $data_provider TEXT, + $stream_id TEXT +) PUBLIC view returns (is_primitive BOOL) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + for $row in SELECT stream_type FROM streams + WHERE id = $stream_ref LIMIT 1 { + return $row.stream_type = 'primitive'; + } --- ERROR('Stream not found: data_provider=' || $data_provider || ' stream_id=' || $stream_id); --- }; - --- /** --- * is_primitive_stream_batch: Checks if multiple streams are primitive in a single query. --- * Returns a table with primitive status for each stream. --- * Only checks streams that exist - does not error on non-existent streams. --- */ --- CREATE OR REPLACE ACTION is_primitive_stream_batch( --- $data_providers TEXT[], --- $stream_ids TEXT[] --- ) PUBLIC view returns table( --- data_provider TEXT, --- stream_id TEXT, --- is_primitive BOOL --- ) { --- -- Lowercase data providers directly using UNNEST for efficiency - --- -- Check that arrays have the same length --- if array_length($data_providers) != array_length($stream_ids) { --- ERROR('Data providers and stream IDs arrays must have the same length'); --- } - --- -- Use UNNEST for optimal performance with direct LOWER operations --- RETURN SELECT --- t.data_provider, --- t.stream_id, --- COALESCE(s.stream_type = 'primitive', false) AS is_primitive --- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) --- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) --- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; --- }; - --- /** --- * get_metadata: Retrieves metadata for a stream with pagination and filtering. --- * Supports ordering by creation time and filtering by key and reference. --- */ --- CREATE OR REPLACE ACTION get_metadata_core( --- $stream_ref INT, --- $key TEXT, --- $ref TEXT, --- $limit INT, --- $offset INT, --- $order_by TEXT --- ) PRIVATE view returns table( --- row_id uuid, --- value_i int, --- value_f NUMERIC(36,18), --- value_b bool, --- value_s TEXT, --- value_ref TEXT, --- created_at INT --- ) { --- -- Set default values if parameters are null --- if $limit IS NULL { --- $limit := 100; --- } --- if $offset IS NULL { --- $offset := 0; --- } --- if $order_by IS NULL { --- $order_by := 'created_at DESC'; --- } - --- RETURN SELECT row_id, --- value_i, --- value_f, --- value_b, --- value_s, --- value_ref, --- created_at --- FROM metadata --- WHERE metadata_key = $key --- AND disabled_at IS NULL --- -- do not use LOWER on value_ref, or it will break the index lookup --- AND ($ref IS NULL OR value_ref = LOWER($ref)) --- AND stream_ref = $stream_ref --- ORDER BY --- CASE WHEN $order_by = 'created_at DESC' THEN created_at END DESC, --- CASE WHEN $order_by = 'created_at ASC' THEN created_at END ASC --- LIMIT $limit OFFSET $offset; --- }; - --- CREATE OR REPLACE ACTION get_metadata( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT, --- $ref TEXT, --- $limit INT, --- $offset INT, --- $order_by TEXT --- ) PUBLIC view returns table( --- row_id uuid, --- value_i int, --- value_f NUMERIC(36,18), --- value_b bool, --- value_s TEXT, --- value_ref TEXT, --- created_at INT --- ) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- for $row in get_metadata_core($stream_ref, $key, $ref, $limit, $offset, $order_by) { --- RETURN NEXT $row.row_id, $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref, $row.created_at; --- } --- }; - --- -- Compatibility wrapper that returns single value from get_metadata_core (for functions expecting single value) --- CREATE OR REPLACE ACTION get_metadata_priv_single( --- $stream_ref INT, --- $key TEXT, --- $ref TEXT --- ) PRIVATE view returns (value TEXT) { --- $result TEXT := NULL; --- for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { --- $result := $row.value_ref; --- } --- RETURN $result; --- }; - --- /** --- * get_latest_metadata: Retrieves the latest metadata for a stream. --- */ --- CREATE OR REPLACE ACTION get_latest_metadata_core( --- $stream_ref INT, --- $key TEXT, --- $ref TEXT --- ) PRIVATE view returns table( --- value_i INT, --- value_f NUMERIC(36,18), --- value_b BOOL, --- value_s TEXT, --- value_ref TEXT --- ) { --- for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { --- RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; --- } --- }; - --- CREATE OR REPLACE ACTION get_latest_metadata( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT, --- $ref TEXT --- ) PUBLIC view returns table( --- value_i INT, --- value_f NUMERIC(36,18), --- value_b BOOL, --- value_s TEXT, --- value_ref TEXT --- ) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); - --- for $row in get_latest_metadata_core($stream_ref, $key, $ref) { --- RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; --- } --- }; - --- /** --- * get_latest_metadata_int: Retrieves the latest metadata value for a stream. --- */ --- CREATE OR REPLACE ACTION get_latest_metadata_int_core( --- $stream_ref INT, --- $key TEXT --- ) PRIVATE view returns (value INT) { --- $result INT := NULL; --- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { --- $result := $row.value_i; --- } --- RETURN $result; --- }; - --- CREATE OR REPLACE ACTION get_latest_metadata_int( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT --- ) PUBLIC view returns (value INT) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- return get_latest_metadata_int_core($stream_ref, $key); --- }; - --- /** --- * get_latest_metadata_ref: Retrieves the latest metadata value for a stream. --- */ --- CREATE OR REPLACE ACTION get_latest_metadata_ref_core( --- $stream_ref INT, --- $key TEXT, --- $ref TEXT --- ) PRIVATE view returns (value TEXT) { --- $result TEXT := NULL; --- for $row in get_latest_metadata_core($stream_ref, $key, $ref) { --- $result := $row.value_ref; --- } --- RETURN $result; --- }; - --- CREATE OR REPLACE ACTION get_latest_metadata_ref( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT, --- $ref TEXT --- ) PUBLIC view returns (value TEXT) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- return get_latest_metadata_ref_core($stream_ref, $key, $ref); --- }; - --- /** --- * get_latest_metadata_bool: Retrieves the latest metadata value for a stream. --- */ --- CREATE OR REPLACE ACTION get_latest_metadata_bool_core( --- $stream_ref INT, --- $key TEXT --- ) PRIVATE view returns (value BOOL) { --- $result BOOL := NULL; --- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { --- $result := $row.value_b; --- } --- RETURN $result; --- }; - --- CREATE OR REPLACE ACTION get_latest_metadata_bool( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT --- ) PUBLIC view returns (value BOOL) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- return get_latest_metadata_bool_core($stream_ref, $key); --- }; - --- /** --- * get_latest_metadata_string: Retrieves the latest metadata value for a stream. --- */ --- CREATE OR REPLACE ACTION get_latest_metadata_string_core( --- $stream_ref INT, --- $key TEXT --- ) PRIVATE view returns (value TEXT) { --- $result TEXT := NULL; --- for $row in get_latest_metadata_core($stream_ref, $key, NULL) { --- $result := $row.value_s; --- } --- RETURN $result; --- }; - --- CREATE OR REPLACE ACTION get_latest_metadata_string( --- $data_provider TEXT, --- $stream_id TEXT, --- $key TEXT --- ) PUBLIC view returns (value TEXT) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); --- return get_latest_metadata_string_core($stream_ref, $key); --- }; - --- /** --- * get_category_streams: Retrieves all streams in a category (composed stream). --- * For primitive streams, returns just the stream itself. --- * For composed streams, recursively traverses taxonomy to find all substreams. --- * It doesn't check for the existence of the substreams, it just returns them. --- */ --- CREATE OR REPLACE ACTION get_category_streams( --- $data_provider TEXT, --- $stream_id TEXT, --- $active_from INT, --- $active_to INT --- ) PUBLIC view returns table(data_provider TEXT, stream_id TEXT) { --- $data_provider := LOWER($data_provider); - --- -- Check if stream exists (get_stream_id returns NULL if stream doesn't exist) --- $stream_ref := get_stream_id($data_provider, $stream_id); --- IF $stream_ref IS NULL { --- ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); --- } - --- -- Always return itself first --- RETURN NEXT $data_provider, $stream_id; - --- -- For primitive streams, just return the stream itself --- if is_primitive_stream($data_provider, $stream_id) == true { --- RETURN; --- } - --- -- Set boundaries for time intervals --- $max_int8 INT := 9223372036854775000; --- $effective_active_from INT := COALESCE($active_from, 0); --- $effective_active_to INT := COALESCE($active_to, $max_int8); - --- -- Get all substreams with proper recursive traversal --- return WITH RECURSIVE substreams AS ( --- /*------------------------------------------------------------------ --- * (1) Base Case: overshadow logic for ($data_provider, $stream_id). --- * - For each distinct start_time, pick the row with the max group_sequence. --- * - next_start is used to define [group_sequence_start, group_sequence_end]. --- *------------------------------------------------------------------*/ --- SELECT --- base.stream_ref, --- base.child_stream_ref, + ERROR('Stream not found: data_provider=' || $data_provider || ' stream_id=' || $stream_id); +}; + +/** + * is_primitive_stream_batch: Checks if multiple streams are primitive in a single query. + * Returns a table with primitive status for each stream. + * Only checks streams that exist - does not error on non-existent streams. + */ +CREATE OR REPLACE ACTION is_primitive_stream_batch( + $data_providers TEXT[], + $stream_ids TEXT[] +) PUBLIC view returns table( + data_provider TEXT, + stream_id TEXT, + is_primitive BOOL +) { + -- Lowercase data providers directly using UNNEST for efficiency + + -- Check that arrays have the same length + if array_length($data_providers) != array_length($stream_ids) { + ERROR('Data providers and stream IDs arrays must have the same length'); + } + + -- Use UNNEST for optimal performance with direct LOWER operations + RETURN SELECT + t.data_provider, + t.stream_id, + COALESCE(s.stream_type = 'primitive', false) AS is_primitive + FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) + LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) + LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; +}; + +/** + * get_metadata: Retrieves metadata for a stream with pagination and filtering. + * Supports ordering by creation time and filtering by key and reference. + */ +CREATE OR REPLACE ACTION get_metadata_core( + $stream_ref INT, + $key TEXT, + $ref TEXT, + $limit INT, + $offset INT, + $order_by TEXT +) PRIVATE view returns table( + row_id uuid, + value_i int, + value_f NUMERIC(36,18), + value_b bool, + value_s TEXT, + value_ref TEXT, + created_at INT +) { + -- Set default values if parameters are null + if $limit IS NULL { + $limit := 100; + } + if $offset IS NULL { + $offset := 0; + } + if $order_by IS NULL { + $order_by := 'created_at DESC'; + } + + RETURN SELECT row_id, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at + FROM metadata + WHERE metadata_key = $key + AND disabled_at IS NULL + -- do not use LOWER on value_ref, or it will break the index lookup + AND ($ref IS NULL OR value_ref = LOWER($ref)) + AND stream_ref = $stream_ref + ORDER BY + CASE WHEN $order_by = 'created_at DESC' THEN created_at END DESC, + CASE WHEN $order_by = 'created_at ASC' THEN created_at END ASC + LIMIT $limit OFFSET $offset; +}; + +CREATE OR REPLACE ACTION get_metadata( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT, + $ref TEXT, + $limit INT, + $offset INT, + $order_by TEXT +) PUBLIC view returns table( + row_id uuid, + value_i int, + value_f NUMERIC(36,18), + value_b bool, + value_s TEXT, + value_ref TEXT, + created_at INT +) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + for $row in get_metadata_core($stream_ref, $key, $ref, $limit, $offset, $order_by) { + RETURN NEXT $row.row_id, $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref, $row.created_at; + } +}; + +-- Compatibility wrapper that returns single value from get_metadata_core (for functions expecting single value) +CREATE OR REPLACE ACTION get_metadata_priv_single( + $stream_ref INT, + $key TEXT, + $ref TEXT +) PRIVATE view returns (value TEXT) { + $result TEXT := NULL; + for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { + $result := $row.value_ref; + } + RETURN $result; +}; + +/** + * get_latest_metadata: Retrieves the latest metadata for a stream. + */ +CREATE OR REPLACE ACTION get_latest_metadata_core( + $stream_ref INT, + $key TEXT, + $ref TEXT +) PRIVATE view returns table( + value_i INT, + value_f NUMERIC(36,18), + value_b BOOL, + value_s TEXT, + value_ref TEXT +) { + for $row in get_metadata_core($stream_ref, $key, $ref, 1, 0, 'created_at DESC') { + RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; + } +}; + +CREATE OR REPLACE ACTION get_latest_metadata( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT, + $ref TEXT +) PUBLIC view returns table( + value_i INT, + value_f NUMERIC(36,18), + value_b BOOL, + value_s TEXT, + value_ref TEXT +) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + + for $row in get_latest_metadata_core($stream_ref, $key, $ref) { + RETURN NEXT $row.value_i, $row.value_f, $row.value_b, $row.value_s, $row.value_ref; + } +}; + +/** + * get_latest_metadata_int: Retrieves the latest metadata value for a stream. + */ +CREATE OR REPLACE ACTION get_latest_metadata_int_core( + $stream_ref INT, + $key TEXT +) PRIVATE view returns (value INT) { + $result INT := NULL; + for $row in get_latest_metadata_core($stream_ref, $key, NULL) { + $result := $row.value_i; + } + RETURN $result; +}; + +CREATE OR REPLACE ACTION get_latest_metadata_int( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT +) PUBLIC view returns (value INT) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + return get_latest_metadata_int_core($stream_ref, $key); +}; + +/** + * get_latest_metadata_ref: Retrieves the latest metadata value for a stream. + */ +CREATE OR REPLACE ACTION get_latest_metadata_ref_core( + $stream_ref INT, + $key TEXT, + $ref TEXT +) PRIVATE view returns (value TEXT) { + $result TEXT := NULL; + for $row in get_latest_metadata_core($stream_ref, $key, $ref) { + $result := $row.value_ref; + } + RETURN $result; +}; + +CREATE OR REPLACE ACTION get_latest_metadata_ref( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT, + $ref TEXT +) PUBLIC view returns (value TEXT) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + return get_latest_metadata_ref_core($stream_ref, $key, $ref); +}; + +/** + * get_latest_metadata_bool: Retrieves the latest metadata value for a stream. + */ +CREATE OR REPLACE ACTION get_latest_metadata_bool_core( + $stream_ref INT, + $key TEXT +) PRIVATE view returns (value BOOL) { + $result BOOL := NULL; + for $row in get_latest_metadata_core($stream_ref, $key, NULL) { + $result := $row.value_b; + } + RETURN $result; +}; + +CREATE OR REPLACE ACTION get_latest_metadata_bool( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT +) PUBLIC view returns (value BOOL) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + return get_latest_metadata_bool_core($stream_ref, $key); +}; + +/** + * get_latest_metadata_string: Retrieves the latest metadata value for a stream. + */ +CREATE OR REPLACE ACTION get_latest_metadata_string_core( + $stream_ref INT, + $key TEXT +) PRIVATE view returns (value TEXT) { + $result TEXT := NULL; + for $row in get_latest_metadata_core($stream_ref, $key, NULL) { + $result := $row.value_s; + } + RETURN $result; +}; + +CREATE OR REPLACE ACTION get_latest_metadata_string( + $data_provider TEXT, + $stream_id TEXT, + $key TEXT +) PUBLIC view returns (value TEXT) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + return get_latest_metadata_string_core($stream_ref, $key); +}; + +/** + * get_category_streams: Retrieves all streams in a category (composed stream). + * For primitive streams, returns just the stream itself. + * For composed streams, recursively traverses taxonomy to find all substreams. + * It doesn't check for the existence of the substreams, it just returns them. + */ +CREATE OR REPLACE ACTION get_category_streams( + $data_provider TEXT, + $stream_id TEXT, + $active_from INT, + $active_to INT +) PUBLIC view returns table(data_provider TEXT, stream_id TEXT) { + $data_provider := LOWER($data_provider); + + -- Check if stream exists (get_stream_id returns NULL if stream doesn't exist) + $stream_ref := get_stream_id($data_provider, $stream_id); + IF $stream_ref IS NULL { + ERROR('Stream does not exist: data_provider=' || $data_provider || ' stream_id=' || $stream_id); + } + + -- Always return itself first + RETURN NEXT $data_provider, $stream_id; + + -- For primitive streams, just return the stream itself + if is_primitive_stream($data_provider, $stream_id) == true { + RETURN; + } + + -- Set boundaries for time intervals + $max_int8 INT := 9223372036854775000; + $effective_active_from INT := COALESCE($active_from, 0); + $effective_active_to INT := COALESCE($active_to, $max_int8); + + -- Get all substreams with proper recursive traversal + return WITH RECURSIVE substreams AS ( + /*------------------------------------------------------------------ + * (1) Base Case: overshadow logic for ($data_provider, $stream_id). + * - For each distinct start_time, pick the row with the max group_sequence. + * - next_start is used to define [group_sequence_start, group_sequence_end]. + *------------------------------------------------------------------*/ + SELECT + base.stream_ref, + base.child_stream_ref, --- -- The interval during which this row is active: --- base.start_time AS group_sequence_start, --- COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end --- FROM ( --- -- Find rows with maximum group_sequence for each start_time --- SELECT --- t.stream_ref, --- t.child_stream_ref, --- t.start_time, --- t.group_sequence, --- MAX(t.group_sequence) OVER ( --- PARTITION BY t.stream_ref, t.start_time --- ) AS max_group_sequence --- FROM taxonomies t --- WHERE t.stream_ref = $stream_ref --- AND t.disabled_at IS NULL --- AND t.start_time <= $effective_active_to --- AND t.start_time >= COALESCE(( --- -- Find the most recent taxonomy at or before effective_active_from --- SELECT t2.start_time --- FROM taxonomies t2 --- WHERE t2.stream_ref = t.stream_ref --- AND t2.disabled_at IS NULL --- AND t2.start_time <= $effective_active_from --- ORDER BY t2.start_time DESC, t2.group_sequence DESC --- LIMIT 1 --- ), 0 --- ) --- ) base --- JOIN ( --- /* Distinct start_times for top-level (dp, sid), used for LEAD() */ --- SELECT --- dt.stream_ref, --- dt.start_time, --- LEAD(dt.start_time) OVER ( --- PARTITION BY dt.stream_ref --- ORDER BY dt.start_time --- ) AS next_start --- FROM ( --- SELECT DISTINCT --- t.stream_ref, --- t.start_time --- FROM taxonomies t --- WHERE t.stream_ref = $stream_ref --- AND t.disabled_at IS NULL --- AND t.start_time <= $effective_active_to --- AND t.start_time >= COALESCE(( --- SELECT t2.start_time --- FROM taxonomies t2 --- WHERE t2.stream_ref = t.stream_ref --- AND t2.disabled_at IS NULL --- AND t2.start_time <= $effective_active_from --- ORDER BY t2.start_time DESC, t2.group_sequence DESC --- LIMIT 1 --- ), 0 --- ) --- ) dt --- ) ot --- ON base.stream_ref = ot.stream_ref --- AND base.start_time = ot.start_time --- WHERE base.group_sequence = base.max_group_sequence - --- UNION - --- /*------------------------------------------------------------------ --- * (2) Recursive Child-Level Overshadow: --- * For each discovered child, gather overshadow rows for that child --- * and produce intervals that overlap the parent's own active interval. --- *------------------------------------------------------------------*/ --- SELECT --- parent.stream_ref, --- child.child_stream_ref, - --- -- Intersection of parent's active interval and child's: --- GREATEST(parent.group_sequence_start, child.start_time) AS group_sequence_start, --- LEAST(parent.group_sequence_end, child.group_sequence_end) AS group_sequence_end --- FROM substreams parent --- JOIN ( --- /* Child overshadow logic, same pattern as above but for child dp/sid. */ --- SELECT --- base.stream_ref, --- base.child_stream_ref, --- base.start_time, --- COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end --- FROM ( --- SELECT --- t.stream_ref, --- t.child_stream_ref, --- t.start_time, --- t.group_sequence, --- MAX(t.group_sequence) OVER ( --- PARTITION BY t.stream_ref, t.start_time --- ) AS max_group_sequence --- FROM taxonomies t --- WHERE t.disabled_at IS NULL --- AND t.start_time <= $effective_active_to --- AND t.start_time >= COALESCE(( --- -- Most recent taxonomy at or before effective_from --- SELECT t2.start_time --- FROM taxonomies t2 --- WHERE t2.stream_ref = t.stream_ref --- AND t2.disabled_at IS NULL --- AND t2.start_time <= $effective_active_from --- ORDER BY t2.start_time DESC, t2.group_sequence DESC --- LIMIT 1 --- ), 0 --- ) --- ) base --- JOIN ( --- /* Distinct start_times at child level */ --- SELECT --- dt.stream_ref, --- dt.start_time, --- LEAD(dt.start_time) OVER ( --- PARTITION BY dt.stream_ref --- ORDER BY dt.start_time --- ) AS next_start --- FROM ( --- SELECT DISTINCT --- t.stream_ref, --- t.start_time --- FROM taxonomies t --- WHERE t.disabled_at IS NULL --- AND t.start_time <= $effective_active_to --- AND t.start_time >= COALESCE(( --- SELECT t2.start_time --- FROM taxonomies t2 --- WHERE t2.stream_ref = t.stream_ref --- AND t2.disabled_at IS NULL --- AND t2.start_time <= $effective_active_from --- ORDER BY t2.start_time DESC, t2.group_sequence DESC --- LIMIT 1 --- ), 0 --- ) --- ) dt --- ) ot --- ON base.stream_ref = ot.stream_ref --- AND base.start_time = ot.start_time --- WHERE base.group_sequence = base.max_group_sequence --- ) child --- ON child.stream_ref = parent.child_stream_ref + -- The interval during which this row is active: + base.start_time AS group_sequence_start, + COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end + FROM ( + -- Find rows with maximum group_sequence for each start_time + SELECT + t.stream_ref, + t.child_stream_ref, + t.start_time, + t.group_sequence, + MAX(t.group_sequence) OVER ( + PARTITION BY t.stream_ref, t.start_time + ) AS max_group_sequence + FROM taxonomies t + WHERE t.stream_ref = $stream_ref + AND t.disabled_at IS NULL + AND t.start_time <= $effective_active_to + AND t.start_time >= COALESCE(( + -- Find the most recent taxonomy at or before effective_active_from + SELECT t2.start_time + FROM taxonomies t2 + WHERE t2.stream_ref = t.stream_ref + AND t2.disabled_at IS NULL + AND t2.start_time <= $effective_active_from + ORDER BY t2.start_time DESC, t2.group_sequence DESC + LIMIT 1 + ), 0 + ) + ) base + JOIN ( + /* Distinct start_times for top-level (dp, sid), used for LEAD() */ + SELECT + dt.stream_ref, + dt.start_time, + LEAD(dt.start_time) OVER ( + PARTITION BY dt.stream_ref + ORDER BY dt.start_time + ) AS next_start + FROM ( + SELECT DISTINCT + t.stream_ref, + t.start_time + FROM taxonomies t + WHERE t.stream_ref = $stream_ref + AND t.disabled_at IS NULL + AND t.start_time <= $effective_active_to + AND t.start_time >= COALESCE(( + SELECT t2.start_time + FROM taxonomies t2 + WHERE t2.stream_ref = t.stream_ref + AND t2.disabled_at IS NULL + AND t2.start_time <= $effective_active_from + ORDER BY t2.start_time DESC, t2.group_sequence DESC + LIMIT 1 + ), 0 + ) + ) dt + ) ot + ON base.stream_ref = ot.stream_ref + AND base.start_time = ot.start_time + WHERE base.group_sequence = base.max_group_sequence + + UNION + + /*------------------------------------------------------------------ + * (2) Recursive Child-Level Overshadow: + * For each discovered child, gather overshadow rows for that child + * and produce intervals that overlap the parent's own active interval. + *------------------------------------------------------------------*/ + SELECT + parent.stream_ref, + child.child_stream_ref, + + -- Intersection of parent's active interval and child's: + GREATEST(parent.group_sequence_start, child.start_time) AS group_sequence_start, + LEAST(parent.group_sequence_end, child.group_sequence_end) AS group_sequence_end + FROM substreams parent + JOIN ( + /* Child overshadow logic, same pattern as above but for child dp/sid. */ + SELECT + base.stream_ref, + base.child_stream_ref, + base.start_time, + COALESCE(ot.next_start, $max_int8) - 1 AS group_sequence_end + FROM ( + SELECT + t.stream_ref, + t.child_stream_ref, + t.start_time, + t.group_sequence, + MAX(t.group_sequence) OVER ( + PARTITION BY t.stream_ref, t.start_time + ) AS max_group_sequence + FROM taxonomies t + WHERE t.disabled_at IS NULL + AND t.start_time <= $effective_active_to + AND t.start_time >= COALESCE(( + -- Most recent taxonomy at or before effective_from + SELECT t2.start_time + FROM taxonomies t2 + WHERE t2.stream_ref = t.stream_ref + AND t2.disabled_at IS NULL + AND t2.start_time <= $effective_active_from + ORDER BY t2.start_time DESC, t2.group_sequence DESC + LIMIT 1 + ), 0 + ) + ) base + JOIN ( + /* Distinct start_times at child level */ + SELECT + dt.stream_ref, + dt.start_time, + LEAD(dt.start_time) OVER ( + PARTITION BY dt.stream_ref + ORDER BY dt.start_time + ) AS next_start + FROM ( + SELECT DISTINCT + t.stream_ref, + t.start_time + FROM taxonomies t + WHERE t.disabled_at IS NULL + AND t.start_time <= $effective_active_to + AND t.start_time >= COALESCE(( + SELECT t2.start_time + FROM taxonomies t2 + WHERE t2.stream_ref = t.stream_ref + AND t2.disabled_at IS NULL + AND t2.start_time <= $effective_active_from + ORDER BY t2.start_time DESC, t2.group_sequence DESC + LIMIT 1 + ), 0 + ) + ) dt + ) ot + ON base.stream_ref = ot.stream_ref + AND base.start_time = ot.start_time + WHERE base.group_sequence = base.max_group_sequence + ) child + ON child.stream_ref = parent.child_stream_ref --- /* Overlap check: child's interval must intersect parent's */ --- WHERE child.start_time <= parent.group_sequence_end --- AND child.group_sequence_end >= parent.group_sequence_start --- ) --- SELECT DISTINCT --- child_dp.address as child_data_provider, --- child_s.stream_id as child_stream_id --- FROM substreams sub --- JOIN streams child_s ON sub.child_stream_ref = child_s.id --- JOIN data_providers child_dp ON child_s.data_provider_id = child_dp.id; --- }; - --- /** --- * stream_exists: Simple check if a stream exists in the database. --- */ --- CREATE OR REPLACE ACTION stream_exists( --- $data_provider TEXT, --- $stream_id TEXT --- ) PUBLIC view returns (result BOOL) { --- $data_provider := LOWER($data_provider); --- $stream_ref := get_stream_id($data_provider, $stream_id); - --- for $row in SELECT 1 FROM streams WHERE id = $stream_ref { --- return true; --- } --- return false; --- }; - --- /** --- * stream_exists_batch_core: Private version that uses stream refs directly. --- * Checks if multiple streams exist using their stream references. --- * Returns false if any stream refs are null (indicating non-existent streams). --- * Returns true only if all streams exist and no stream refs are null. --- */ --- CREATE OR REPLACE ACTION stream_exists_batch_core( --- $stream_refs INT[] --- ) PRIVATE VIEW RETURNS (result BOOL) { --- -- Use UNNEST for efficient batch processing --- for $row in SELECT CASE --- WHEN EXISTS ( --- SELECT 1 FROM UNNEST($stream_refs) AS t(stream_ref) WHERE t.stream_ref IS NULL --- ) THEN false --- ELSE NOT EXISTS ( --- SELECT 1 --- FROM UNNEST($stream_refs) AS t(stream_ref) --- LEFT JOIN streams s ON s.id = t.stream_ref --- WHERE s.id IS NULL --- AND t.stream_ref IS NOT NULL --- ) --- END AS result --- FROM (SELECT 1) dummy { --- return $row.result; --- } --- return false; --- }; - --- /** --- * is_primitive_stream_batch_core: Private version that uses stream refs directly. --- * Checks if multiple streams are primitive using their stream references. --- * Returns false if any stream refs are null (indicating non-existent streams). --- * Returns true only if all streams exist and are primitive. --- */ --- CREATE OR REPLACE ACTION is_primitive_stream_batch_core( --- $stream_refs INT[] --- ) PRIVATE VIEW RETURNS (result BOOL) { --- -- Use UNNEST for optimal performance - direct array processing without recursion --- for $row in SELECT CASE --- WHEN EXISTS ( --- SELECT 1 --- FROM UNNEST($stream_refs) AS t(stream_ref) --- WHERE t.stream_ref IS NULL --- ) THEN false --- ELSE NOT EXISTS ( --- SELECT 1 --- FROM UNNEST($stream_refs) AS t(stream_ref) --- JOIN streams s ON s.id = t.stream_ref --- WHERE s.stream_type != 'primitive' --- AND t.stream_ref IS NOT NULL --- ) --- END AS result --- FROM (SELECT 1) dummy { --- return $row.result; --- } --- return false; --- }; - --- /** --- * stream_exists_batch: Checks existence of multiple streams in a single query. --- * Returns a table with existence status for each stream. --- */ --- CREATE OR REPLACE ACTION stream_exists_batch( --- $data_providers TEXT[], --- $stream_ids TEXT[] --- ) PUBLIC view returns table( --- data_provider TEXT, --- stream_id TEXT, --- stream_exists BOOL --- ) { --- -- Lowercase data providers directly using UNNEST for efficiency - --- -- Check that arrays have the same length --- if array_length($data_providers) != array_length($stream_ids) { --- ERROR('Data providers and stream IDs arrays must have the same length'); --- } - --- -- Use UNNEST for optimal performance with direct LOWER operations --- RETURN SELECT --- t.data_provider, --- t.stream_id, --- CASE WHEN s.data_provider IS NOT NULL THEN true ELSE false END AS stream_exists --- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) --- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) --- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; --- }; - --- CREATE OR REPLACE ACTION transfer_stream_ownership( --- $data_provider TEXT, --- $stream_id TEXT, --- $new_owner TEXT --- ) PUBLIC { --- $data_provider := LOWER($data_provider); --- $new_owner := LOWER($new_owner); --- $lower_caller := LOWER(@caller); --- $stream_ref := get_stream_id($data_provider, $stream_id); - --- if !is_stream_owner($data_provider, $stream_id, $lower_caller) { --- ERROR('Only stream owner can transfer ownership'); --- } - --- -- Check if new owner is a valid ethereum address --- if NOT check_ethereum_address($new_owner) { --- ERROR('Invalid new owner address. Must be a valid Ethereum address: ' || $new_owner); --- } - --- -- Update the stream_owner metadata --- UPDATE metadata SET value_ref = LOWER($new_owner) --- WHERE metadata_key = 'stream_owner' --- AND stream_ref = $stream_ref; --- }; - --- /** --- * filter_streams_by_existence: Filters streams based on existence. --- * Can return either existing or non-existing streams based on existing_only flag. --- * Takes arrays of data providers and stream IDs as input. --- * Uses efficient WITH RECURSIVE pattern for batch processing. --- */ --- CREATE OR REPLACE ACTION filter_streams_by_existence( --- $data_providers TEXT[], --- $stream_ids TEXT[], --- $existing_only BOOL --- ) PUBLIC view returns table( --- data_provider TEXT, --- stream_id TEXT --- ) { --- -- Lowercase data providers directly using UNNEST for efficiency - --- -- default to return existing streams --- if $existing_only IS NULL { --- $existing_only := true; --- } + /* Overlap check: child's interval must intersect parent's */ + WHERE child.start_time <= parent.group_sequence_end + AND child.group_sequence_end >= parent.group_sequence_start + ) + SELECT DISTINCT + child_dp.address as child_data_provider, + child_s.stream_id as child_stream_id + FROM substreams sub + JOIN streams child_s ON sub.child_stream_ref = child_s.id + JOIN data_providers child_dp ON child_s.data_provider_id = child_dp.id; +}; + +/** + * stream_exists: Simple check if a stream exists in the database. + */ +CREATE OR REPLACE ACTION stream_exists( + $data_provider TEXT, + $stream_id TEXT +) PUBLIC view returns (result BOOL) { + $data_provider := LOWER($data_provider); + $stream_ref := get_stream_id($data_provider, $stream_id); + + for $row in SELECT 1 FROM streams WHERE id = $stream_ref { + return true; + } + return false; +}; + +/** + * stream_exists_batch_core: Private version that uses stream refs directly. + * Checks if multiple streams exist using their stream references. + * Returns false if any stream refs are null (indicating non-existent streams). + * Returns true only if all streams exist and no stream refs are null. + */ +CREATE OR REPLACE ACTION stream_exists_batch_core( + $stream_refs INT[] +) PRIVATE VIEW RETURNS (result BOOL) { + -- Use UNNEST for efficient batch processing + for $row in SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM UNNEST($stream_refs) AS t(stream_ref) WHERE t.stream_ref IS NULL + ) THEN false + ELSE NOT EXISTS ( + SELECT 1 + FROM UNNEST($stream_refs) AS t(stream_ref) + LEFT JOIN streams s ON s.id = t.stream_ref + WHERE s.id IS NULL + AND t.stream_ref IS NOT NULL + ) + END AS result + FROM (SELECT 1) dummy { + return $row.result; + } + return false; +}; + +/** + * is_primitive_stream_batch_core: Private version that uses stream refs directly. + * Checks if multiple streams are primitive using their stream references. + * Returns false if any stream refs are null (indicating non-existent streams). + * Returns true only if all streams exist and are primitive. + */ +CREATE OR REPLACE ACTION is_primitive_stream_batch_core( + $stream_refs INT[] +) PRIVATE VIEW RETURNS (result BOOL) { + -- Use UNNEST for optimal performance - direct array processing without recursion + for $row in SELECT CASE + WHEN EXISTS ( + SELECT 1 + FROM UNNEST($stream_refs) AS t(stream_ref) + WHERE t.stream_ref IS NULL + ) THEN false + ELSE NOT EXISTS ( + SELECT 1 + FROM UNNEST($stream_refs) AS t(stream_ref) + JOIN streams s ON s.id = t.stream_ref + WHERE s.stream_type != 'primitive' + AND t.stream_ref IS NOT NULL + ) + END AS result + FROM (SELECT 1) dummy { + return $row.result; + } + return false; +}; + +/** + * stream_exists_batch: Checks existence of multiple streams in a single query. + * Returns a table with existence status for each stream. + */ +CREATE OR REPLACE ACTION stream_exists_batch( + $data_providers TEXT[], + $stream_ids TEXT[] +) PUBLIC view returns table( + data_provider TEXT, + stream_id TEXT, + stream_exists BOOL +) { + -- Lowercase data providers directly using UNNEST for efficiency + + -- Check that arrays have the same length + if array_length($data_providers) != array_length($stream_ids) { + ERROR('Data providers and stream IDs arrays must have the same length'); + } + + -- Use UNNEST for optimal performance with direct LOWER operations + RETURN SELECT + t.data_provider, + t.stream_id, + CASE WHEN s.data_provider IS NOT NULL THEN true ELSE false END AS stream_exists + FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) + LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) + LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id; +}; + +CREATE OR REPLACE ACTION transfer_stream_ownership( + $data_provider TEXT, + $stream_id TEXT, + $new_owner TEXT +) PUBLIC { + $data_provider := LOWER($data_provider); + $new_owner := LOWER($new_owner); + $lower_caller := LOWER(@caller); + $stream_ref := get_stream_id($data_provider, $stream_id); + + if !is_stream_owner($data_provider, $stream_id, $lower_caller) { + ERROR('Only stream owner can transfer ownership'); + } + + -- Check if new owner is a valid ethereum address + if NOT check_ethereum_address($new_owner) { + ERROR('Invalid new owner address. Must be a valid Ethereum address: ' || $new_owner); + } + + -- Update the stream_owner metadata + UPDATE metadata SET value_ref = LOWER($new_owner) + WHERE metadata_key = 'stream_owner' + AND stream_ref = $stream_ref; +}; + +/** + * filter_streams_by_existence: Filters streams based on existence. + * Can return either existing or non-existing streams based on existing_only flag. + * Takes arrays of data providers and stream IDs as input. + * Uses efficient WITH RECURSIVE pattern for batch processing. + */ +CREATE OR REPLACE ACTION filter_streams_by_existence( + $data_providers TEXT[], + $stream_ids TEXT[], + $existing_only BOOL +) PUBLIC view returns table( + data_provider TEXT, + stream_id TEXT +) { + -- Lowercase data providers directly using UNNEST for efficiency + + -- default to return existing streams + if $existing_only IS NULL { + $existing_only := true; + } --- -- Check that arrays have the same length --- if array_length($data_providers) != array_length($stream_ids) { --- ERROR('Data providers and stream IDs arrays must have the same length'); --- } + -- Check that arrays have the same length + if array_length($data_providers) != array_length($stream_ids) { + ERROR('Data providers and stream IDs arrays must have the same length'); + } --- -- Use UNNEST for efficient batch processing --- RETURN SELECT --- t.data_provider, --- t.stream_id --- FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) --- LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) --- LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id --- WHERE (CASE WHEN s.id IS NOT NULL THEN true ELSE false END) = $existing_only; --- }; - --- CREATE OR REPLACE ACTION list_streams( --- $data_provider TEXT, --- $limit INT, --- $offset INT, --- $order_by TEXT, --- $block_height INT --- ) PUBLIC view returns table( --- data_provider TEXT, --- stream_id TEXT, --- stream_type TEXT, --- created_at INT8 --- ) { --- $data_provider := LOWER($data_provider); - --- if $limit > 5000 { --- ERROR('Limit exceeds maximum allowed value of 5000'); --- } --- if $limit IS NULL { --- $limit := 5000; --- } --- if $limit == 0 { --- $limit := 5000; --- } --- if $offset IS NULL { --- $offset := 0; --- } --- if $order_by IS NULL { --- $order_by := 'created_at DESC'; --- } --- if $order_by == '' { --- $order_by := 'created_at DESC'; --- } - --- RETURN SELECT --- dp.address as data_provider, --- s.stream_id, --- s.stream_type, --- s.created_at --- FROM streams s --- JOIN data_providers dp ON s.data_provider_id = dp.id --- -- do not use LOWER on dp.address, or it will break the index lookup --- WHERE ($data_provider IS NULL OR $data_provider = '' OR dp.address = LOWER($data_provider)) --- AND s.created_at > $block_height --- ORDER BY --- CASE WHEN $order_by = 'created_at DESC' THEN s.created_at END DESC, --- CASE WHEN $order_by = 'created_at ASC' THEN s.created_at END ASC, --- CASE WHEN $order_by = 'stream_id ASC' THEN stream_id END ASC, --- CASE WHEN $order_by = 'stream_id DESC' THEN stream_id END DESC, --- CASE WHEN $order_by = 'stream_type ASC' THEN stream_type END ASC, --- CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC --- LIMIT $limit OFFSET $offset; --- }; + -- Use UNNEST for efficient batch processing + RETURN SELECT + t.data_provider, + t.stream_id + FROM UNNEST($data_providers, $stream_ids) AS t(data_provider, stream_id) + LEFT JOIN data_providers dp ON dp.address = LOWER(t.data_provider) + LEFT JOIN streams s ON s.data_provider_id = dp.id AND s.stream_id = t.stream_id + WHERE (CASE WHEN s.id IS NOT NULL THEN true ELSE false END) = $existing_only; +}; + +CREATE OR REPLACE ACTION list_streams( + $data_provider TEXT, + $limit INT, + $offset INT, + $order_by TEXT, + $block_height INT +) PUBLIC view returns table( + data_provider TEXT, + stream_id TEXT, + stream_type TEXT, + created_at INT8 +) { + $data_provider := LOWER($data_provider); + + if $limit > 5000 { + ERROR('Limit exceeds maximum allowed value of 5000'); + } + if $limit IS NULL { + $limit := 5000; + } + if $limit == 0 { + $limit := 5000; + } + if $offset IS NULL { + $offset := 0; + } + if $order_by IS NULL { + $order_by := 'created_at DESC'; + } + if $order_by == '' { + $order_by := 'created_at DESC'; + } + + RETURN SELECT + dp.address as data_provider, + s.stream_id, + s.stream_type, + s.created_at + FROM streams s + JOIN data_providers dp ON s.data_provider_id = dp.id + -- do not use LOWER on dp.address, or it will break the index lookup + WHERE ($data_provider IS NULL OR $data_provider = '' OR dp.address = LOWER($data_provider)) + AND s.created_at > $block_height + ORDER BY + CASE WHEN $order_by = 'created_at DESC' THEN s.created_at END DESC, + CASE WHEN $order_by = 'created_at ASC' THEN s.created_at END ASC, + CASE WHEN $order_by = 'stream_id ASC' THEN stream_id END ASC, + CASE WHEN $order_by = 'stream_id DESC' THEN stream_id END DESC, + CASE WHEN $order_by = 'stream_type ASC' THEN stream_type END ASC, + CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC + LIMIT $limit OFFSET $offset; +}; /** * list_metadata_by_height: Queries metadata within a specific block height range. From c8c6aa78fa1fa45aa0bfea5980732abce3acf145 Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 22:40:50 +0700 Subject: [PATCH 3/7] feat: test --- tests/streams/query/metadata_test.go | 59 +++++++++++++++++++++++++- tests/streams/utils/procedure/types.go | 2 +- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/streams/query/metadata_test.go b/tests/streams/query/metadata_test.go index d02ea7715..06d2c715b 100644 --- a/tests/streams/query/metadata_test.go +++ b/tests/streams/query/metadata_test.go @@ -14,6 +14,7 @@ import ( testutils "github.com/trufnetwork/node/tests/streams/utils" "github.com/trufnetwork/node/tests/streams/utils/procedure" "github.com/trufnetwork/node/tests/streams/utils/setup" + "github.com/trufnetwork/node/tests/streams/utils/table" "github.com/trufnetwork/sdk-go/core/types" "github.com/trufnetwork/sdk-go/core/util" ) @@ -29,14 +30,19 @@ var ( } ) -// TestQUERY04MetadataInsertionAndRetrieval tests the insertion and retrieval of metadata +// TestQUERY04Metadata tests the insertion and retrieval of metadata // for both primitive and composed streams. Also tests disabling metadata and attempting to retrieve it. -func TestQUERY04MetadataInsertionAndRetrieval(t *testing.T) { +func TestQUERY04Metadata(t *testing.T) { kwilTesting.RunSchemaTest(t, kwilTesting.SchemaTest{ Name: "metadata_insertion_and_retrieval", FunctionTests: []kwilTesting.TestFunc{ WithMetadataTestSetup(testMetadataInsertionAndRetrieval(t, primitiveContractInfo)), WithMetadataTestSetup(testMetadataInsertionThenDisableAndRetrieval(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeight(t, primitiveContractInfo)), + // WithMetadataTestSetup(testListMetadataByHeightNoKey(t, primitiveContractInfo)), + // WithMetadataTestSetup(testListMetadataByHeightPagination(t, primitiveContractInfo)), + // WithMetadataTestSetup(testListMetadataByHeightInvalidRange(t, primitiveContractInfo)), + // WithMetadataTestSetup(testListMetadataByHeightInvalidPagination(t, primitiveContractInfo)), }, SeedScripts: migrations.GetSeedScriptPaths(), }, testutils.GetTestOptions()) @@ -161,3 +167,52 @@ func testMetadataInsertionThenDisableAndRetrieval(t *testing.T, contractInfo set return nil } } + +func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {"test_metadata", "1", "int"}, + {"test_metadata", "0", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: "test_metadata", + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + expected := ` + | stream_ref | row_id | value_i | value_f | value_b | value_s | value_ref | created_at | + |---------------|-----------|---------------------|-----------------|--------|------------|----------------|------------|------------|------------| + | 1 | 1d6c118f-0f95-5a67-91d6-9ae777aaf5f5 | 1 | | | | | 1 | + | 1 | ba084c0a-6558-56d6-bea6-af6ebfe72fd5 | 0 | | | | | 1 |` + + table.AssertResultRowsEqualMarkdownTable(t, table.AssertResultRowsEqualMarkdownTableInput{ + Actual: result, + Expected: expected, + }) + + return nil + } +} diff --git a/tests/streams/utils/procedure/types.go b/tests/streams/utils/procedure/types.go index 19b8322bf..37871636f 100644 --- a/tests/streams/utils/procedure/types.go +++ b/tests/streams/utils/procedure/types.go @@ -163,7 +163,7 @@ type GetTaxonomiesForStreamsInput struct { type ListMetadataByHeightInput struct { Platform *kwilTesting.Platform Key string - Value string + Value *string FromHeight *int64 ToHeight *int64 Limit *int From ddca28ec9ed80bf9ec4ea863a353594877d2f4c2 Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 22:58:19 +0700 Subject: [PATCH 4/7] feat: complete tests --- tests/streams/query/metadata_test.go | 230 ++++++++++++++++++++++++++- 1 file changed, 223 insertions(+), 7 deletions(-) diff --git a/tests/streams/query/metadata_test.go b/tests/streams/query/metadata_test.go index 06d2c715b..5e9559e5e 100644 --- a/tests/streams/query/metadata_test.go +++ b/tests/streams/query/metadata_test.go @@ -3,6 +3,7 @@ package tests import ( "context" "fmt" + "strings" "testing" "github.com/pkg/errors" @@ -39,10 +40,10 @@ func TestQUERY04Metadata(t *testing.T) { WithMetadataTestSetup(testMetadataInsertionAndRetrieval(t, primitiveContractInfo)), WithMetadataTestSetup(testMetadataInsertionThenDisableAndRetrieval(t, primitiveContractInfo)), WithMetadataTestSetup(testListMetadataByHeight(t, primitiveContractInfo)), - // WithMetadataTestSetup(testListMetadataByHeightNoKey(t, primitiveContractInfo)), - // WithMetadataTestSetup(testListMetadataByHeightPagination(t, primitiveContractInfo)), - // WithMetadataTestSetup(testListMetadataByHeightInvalidRange(t, primitiveContractInfo)), - // WithMetadataTestSetup(testListMetadataByHeightInvalidPagination(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightNoKey(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightPagination(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightInvalidRange(t, primitiveContractInfo)), + WithMetadataTestSetup(testListMetadataByHeightInvalidPagination(t, primitiveContractInfo)), }, SeedScripts: migrations.GetSeedScriptPaths(), }, testutils.GetTestOptions()) @@ -170,13 +171,14 @@ func testMetadataInsertionThenDisableAndRetrieval(t *testing.T, contractInfo set func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" metadataItems := []struct { Key string Value string ValType string }{ - {"test_metadata", "1", "int"}, - {"test_metadata", "0", "int"}, + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, } for _, item := range metadataItems { @@ -195,7 +197,7 @@ func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilT result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ Platform: platform, - Key: "test_metadata", + Key: metadataKey, Height: 1, }) if err != nil { @@ -216,3 +218,217 @@ func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilT return nil } } + +func testListMetadataByHeightNoKey(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if (len(result) > 0) { // should return no rows + return errors.Wrapf(err, "expected empty results") + } + + return nil + } +} + +func testListMetadataByHeightPagination(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + limit := 2 + offset := 0 + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Offset: &offset, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if len(result) != 2 { + return errors.Errorf("expected 2 results, got %d", len(result)) + } + + offset = 2 + result, err = procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Offset: &offset, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error listing metadata") + } + + if len(result) != 1 { + return errors.Errorf("expected 1 result, got %d", len(result)) + } + + return nil + } +} + +func testListMetadataByHeightInvalidRange(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + fromHeight := int64(10) + toHeight := int64(5) + _, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + FromHeight: &fromHeight, + ToHeight: &toHeight, + Height: 1, + }) + + if err == nil { + return errors.New("expected error for invalid height range (from_height > to_height), but got none") + } + + expectedError := "Invalid height range" + if !strings.Contains(err.Error(), expectedError) { + return errors.Errorf("expected error message to contain '%s', got: %s", expectedError, err.Error()) + } + + return nil + } +} + +func testListMetadataByHeightInvalidPagination(t *testing.T, contractInfo setup.StreamInfo) kwilTesting.TestFunc { + return func(ctx context.Context, platform *kwilTesting.Platform) error { + metadataKey := "test_metadata" + metadataItems := []struct { + Key string + Value string + ValType string + }{ + {metadataKey, "1", "int"}, + {metadataKey, "0", "int"}, + {metadataKey, "1", "int"}, + } + + for _, item := range metadataItems { + err := procedure.InsertMetadata(ctx, procedure.InsertMetadataInput{ + Platform: platform, + Locator: contractInfo.Locator, + Key: item.Key, + Value: item.Value, + ValType: item.ValType, + Height: 1, + }) + if err != nil { + return errors.Wrapf(err, "error inserting metadata with key %s", item.Key) + } + } + + // negative limit + limit := -10 + result, err := procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Limit: &limit, + Height: 1, + }) + if err != nil { + return errors.Wrap(err, "unexpected error with negative limit") + } + + if len(result) != 0 { + return errors.Errorf("expected empty results with negative limit (converted to 0), got %d results", len(result)) + } + + negativeOffset := -5 + result, err = procedure.ListMetadataByHeight(ctx, procedure.ListMetadataByHeightInput{ + Platform: platform, + Key: metadataKey, + Offset: &negativeOffset, + Height: 1, + }) + if err != nil { + return errors.Wrap(err, "unexpected error with negative offset") + } + + return nil + } +} From 304efdffe80d1e2582a51feffb632d14f0bfaf3c Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 23:00:37 +0700 Subject: [PATCH 5/7] chore: put the new metadata action into separate migration file --- internal/migrations/001-common-actions.sql | 81 -------------------- internal/migrations/021-metadata-actions.sql | 80 +++++++++++++++++++ 2 files changed, 80 insertions(+), 81 deletions(-) create mode 100644 internal/migrations/021-metadata-actions.sql diff --git a/internal/migrations/001-common-actions.sql b/internal/migrations/001-common-actions.sql index d331c9db3..6c243d3ac 100644 --- a/internal/migrations/001-common-actions.sql +++ b/internal/migrations/001-common-actions.sql @@ -1320,84 +1320,3 @@ CREATE OR REPLACE ACTION list_streams( CASE WHEN $order_by = 'stream_type DESC' THEN stream_type END DESC LIMIT $limit OFFSET $offset; }; - -/** - * list_metadata_by_height: Queries metadata within a specific block height range. - * Supports pagination. - * Supports filtering by key and ref. - * - * Parameters: - * $key: filter by metadata key. - *. $ref: filter by value reference. - * $from_height: Start height (inclusive). If NULL, uses earliest available. - * $to_height: End height (inclusive). If NULL, uses current height. - * $limit: Maximum number of results to return. - * $offset: Number of results to skip for pagination. - * - * Returns: - * Table with metadata entries matching the criteria. - */ -CREATE OR REPLACE ACTION list_metadata_by_height( - $key TEXT, - $ref TEXT, - $from_height INT8, - $to_height INT8, - $limit INT, - $offset INT -) PUBLIC view returns table( - stream_ref INT, - row_id uuid, - value_i INT, - value_f NUMERIC(36,18), - value_b bool, - value_s TEXT, - value_ref TEXT, - created_at INT8 -) { - -- Set defaults for pagination and validate values - if $limit IS NULL { - $limit := 1000; - } - if $offset IS NULL { - $offset := 0; - } - - -- Ensure non-negative values for PostgreSQL compatibility - if $limit < 0 { - $limit := 0; - } - if $offset < 0 { - $offset := 0; - } - - -- Get current block height for default behavior - $current_block INT8 := @height; - - -- Determine effective height range, if none given, get all metadata - $effective_from INT8 := COALESCE($from_height, 0); - $effective_to INT8 := COALESCE($to_height, $current_block); - - -- Validate height range - if $effective_from > $effective_to { - ERROR('Invalid height range: from_height (' || $effective_from::TEXT || ') > to_height (' || $effective_to::TEXT || ')'); - } - - RETURN SELECT - stream_ref, - row_id, - value_i, - value_f, - value_b, - value_s, - value_ref, - created_at - FROM metadata - WHERE metadata_key = $key - AND disabled_at IS NULL - -- do not use LOWER on value_ref, or it will break the index lookup - AND ($ref IS NULL OR value_ref = LOWER($ref)) - AND created_at >= $effective_from - AND created_at <= $effective_to - ORDER BY created_at ASC - LIMIT $limit OFFSET $offset; -}; \ No newline at end of file diff --git a/internal/migrations/021-metadata-actions.sql b/internal/migrations/021-metadata-actions.sql new file mode 100644 index 000000000..7480649c8 --- /dev/null +++ b/internal/migrations/021-metadata-actions.sql @@ -0,0 +1,80 @@ +/** + * list_metadata_by_height: Queries metadata within a specific block height range. + * Supports pagination. + * Supports filtering by key and ref. + * + * Parameters: + * $key: filter by metadata key. + *. $ref: filter by value reference. + * $from_height: Start height (inclusive). If NULL, uses earliest available. + * $to_height: End height (inclusive). If NULL, uses current height. + * $limit: Maximum number of results to return. + * $offset: Number of results to skip for pagination. + * + * Returns: + * Table with metadata entries matching the criteria. + */ +CREATE OR REPLACE ACTION list_metadata_by_height( + $key TEXT, + $ref TEXT, + $from_height INT8, + $to_height INT8, + $limit INT, + $offset INT +) PUBLIC view returns table( + stream_ref INT, + row_id uuid, + value_i INT, + value_f NUMERIC(36,18), + value_b bool, + value_s TEXT, + value_ref TEXT, + created_at INT8 +) { + -- Set defaults for pagination and validate values + if $limit IS NULL { + $limit := 1000; + } + if $offset IS NULL { + $offset := 0; + } + + -- Ensure non-negative values for PostgreSQL compatibility + if $limit < 0 { + $limit := 0; + } + if $offset < 0 { + $offset := 0; + } + + -- Get current block height for default behavior + $current_block INT8 := @height; + + -- Determine effective height range, if none given, get all metadata + $effective_from INT8 := COALESCE($from_height, 0); + $effective_to INT8 := COALESCE($to_height, $current_block); + + -- Validate height range + if $effective_from > $effective_to { + ERROR('Invalid height range: from_height (' || $effective_from::TEXT || ') > to_height (' || $effective_to::TEXT || ')'); + } + + RETURN SELECT + stream_ref, + row_id, + value_i, + value_f, + value_b, + value_s, + value_ref, + created_at + FROM metadata + WHERE metadata_key = $key + AND disabled_at IS NULL + -- do not use LOWER on value_ref, or it will break the index lookup + AND ($ref IS NULL OR value_ref = LOWER($ref)) + AND created_at >= $effective_from + AND created_at <= $effective_to + ORDER BY created_at ASC + LIMIT $limit OFFSET $offset; +}; \ No newline at end of file From 71e8c209c446c67d134e176f47577e6646976acf Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Thu, 4 Sep 2025 23:50:28 +0700 Subject: [PATCH 6/7] chore: add column exclusion for table markdown --- tests/streams/query/metadata_test.go | 7 ++-- tests/streams/utils/table/assert.go | 54 ++++++++++++++++++---------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/tests/streams/query/metadata_test.go b/tests/streams/query/metadata_test.go index 5e9559e5e..22819cadb 100644 --- a/tests/streams/query/metadata_test.go +++ b/tests/streams/query/metadata_test.go @@ -205,14 +205,15 @@ func testListMetadataByHeight(t *testing.T, contractInfo setup.StreamInfo) kwilT } expected := ` - | stream_ref | row_id | value_i | value_f | value_b | value_s | value_ref | created_at | + | stream_ref | value_i | value_f | value_b | value_s | value_ref | created_at | |---------------|-----------|---------------------|-----------------|--------|------------|----------------|------------|------------|------------| - | 1 | 1d6c118f-0f95-5a67-91d6-9ae777aaf5f5 | 1 | | | | | 1 | - | 1 | ba084c0a-6558-56d6-bea6-af6ebfe72fd5 | 0 | | | | | 1 |` + | 1 | 0 | | | | | 1 | + | 1 | 1 | | | | | 1 |` table.AssertResultRowsEqualMarkdownTable(t, table.AssertResultRowsEqualMarkdownTableInput{ Actual: result, Expected: expected, + ExcludedColumns: []int{1}, }) return nil diff --git a/tests/streams/utils/table/assert.go b/tests/streams/utils/table/assert.go index 17ba5e00b..5f2c1b96a 100644 --- a/tests/streams/utils/table/assert.go +++ b/tests/streams/utils/table/assert.go @@ -13,6 +13,7 @@ type AssertResultRowsEqualMarkdownTableInput struct { Expected string ColumnTransformers map[string]func(string) string SortColumns []string + ExcludedColumns []int // drop only from actual } func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqualMarkdownTableInput) { @@ -21,16 +22,13 @@ func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqua t.Fatalf("error parsing expected markdown table: %v", err) } - // clear empty rows, because we won't get those from answer, but - // tests might include it just to be explicit about what is being tested + // Transform expected rows (apply transformers, keep all columns) expected := [][]string{} for _, row := range expectedTable.Rows { if row[1] != "" { transformedRow := make([]string, len(row)) for colIdx, value := range row { - // Get the column name from headers colName := expectedTable.Headers[colIdx] - // Apply transformer if one exists for this column if transformer, exists := input.ColumnTransformers[colName]; exists && transformer != nil { transformedRow[colIdx] = transformer(value) } else { @@ -41,40 +39,60 @@ func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqua } } + // Build a set of excluded column positions for actual + excludedIdx := make(map[int]struct{}) + for _, idx := range input.ExcludedColumns { + excludedIdx[idx] = struct{}{} + } + + // Transform actual rows (drop excluded columns) actualInStrings := [][]string{} for _, row := range input.Actual { actualRow := []string{} - for _, column := range row { - actualRow = append(actualRow, column) + for colIdx, value := range row { + if _, skip := excludedIdx[colIdx]; skip { + continue + } + actualRow = append(actualRow, value) } actualInStrings = append(actualInStrings, actualRow) } // Sort both expected and actual data if sort columns are specified if len(input.SortColumns) > 0 { - // Create maps of column name to index for sorting - headerIndexMap := make(map[string]int) - for i, header := range expectedTable.Headers { - headerIndexMap[header] = i + // Build mapping from header name -> index in expected + expectedHeaderMap := make(map[string]int) + for i, h := range expectedTable.Headers { + expectedHeaderMap[h] = i + } + + // Build mapping from header name -> index in actual (after dropping columns) + actualHeaderMap := make(map[string]int) + actualColIdx := 0 + for i := range expectedTable.Headers { + if _, skip := excludedIdx[i]; skip { + continue + } + actualHeaderMap[expectedTable.Headers[i]] = actualColIdx + actualColIdx++ } - // Create a custom sort function that can sort by multiple columns - sortFunc := func(rows [][]string) func(i, j int) bool { + // Sort function + sortFunc := func(rows [][]string, headerMap map[string]int) func(i, j int) bool { return func(i, j int) bool { for _, colName := range input.SortColumns { - if idx, ok := headerIndexMap[colName]; ok && idx < len(rows[i]) && idx < len(rows[j]) { + if idx, ok := headerMap[colName]; ok && idx < len(rows[i]) && idx < len(rows[j]) { if rows[i][idx] != rows[j][idx] { - return rows[i][idx] < rows[j][idx] + return rows[i][idx] < rows[j][idx] // ascending } } } - return false // If all sort columns are equal, maintain original order + return false } } - // Sort both expected and actual results - sort.SliceStable(expected, sortFunc(expected)) - sort.SliceStable(actualInStrings, sortFunc(actualInStrings)) + sort.SliceStable(expected, sortFunc(expected, expectedHeaderMap)) + sort.SliceStable(actualInStrings, sortFunc(actualInStrings, actualHeaderMap)) } assert.Equal(t, expected, actualInStrings, "Result rows do not match expected markdown table") From d69f9d59da786454a795fb5a2fe28c154eae40d2 Mon Sep 17 00:00:00 2001 From: williamrusdyputra Date: Fri, 5 Sep 2025 00:30:53 +0700 Subject: [PATCH 7/7] chore: update assert --- tests/streams/utils/table/assert.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/streams/utils/table/assert.go b/tests/streams/utils/table/assert.go index 5f2c1b96a..6cf892573 100644 --- a/tests/streams/utils/table/assert.go +++ b/tests/streams/utils/table/assert.go @@ -66,15 +66,9 @@ func AssertResultRowsEqualMarkdownTable(t *testing.T, input AssertResultRowsEqua expectedHeaderMap[h] = i } - // Build mapping from header name -> index in actual (after dropping columns) actualHeaderMap := make(map[string]int) - actualColIdx := 0 - for i := range expectedTable.Headers { - if _, skip := excludedIdx[i]; skip { - continue - } - actualHeaderMap[expectedTable.Headers[i]] = actualColIdx - actualColIdx++ + for i, h := range expectedTable.Headers { + actualHeaderMap[h] = i } // Sort function