diff --git a/oracle/src/assets.rs b/oracle/src/assets.rs index 844fadb..27fa29f 100644 --- a/oracle/src/assets.rs +++ b/oracle/src/assets.rs @@ -27,16 +27,7 @@ pub fn load_all_assets(e: &Env) -> Vec { // Load asset index pub fn resolve_asset_index(e: &Env, asset: &Asset) -> Option { - let index: Option; - match asset { - Asset::Stellar(address) => { - index = e.storage().instance().get(&address); - } - Asset::Other(symbol) => { - index = e.storage().instance().get(&symbol); - } - } - index + load_all_assets(e).first_index_of(asset) } // Add assets to the oracle @@ -49,10 +40,9 @@ pub fn add_assets(e: &Env, assets: Vec, initial_expiration_period: u32) { //for each new asset for asset in assets.iter() { //check if the asset has been already added - if resolve_asset_index(e, &asset).is_some() { + if asset_list.first_index_of(&asset).is_some() { panic_with_error!(&e, Error::AssetAlreadyExists); } - set_asset_index(e, &asset, asset_list.len()); asset_list.push_back(asset); //update expiration records expiration.push_back(expiration_timestamp); @@ -158,16 +148,3 @@ fn load_expiration_records(e: &Env) -> Vec { fn set_expirations_records(e: &Env, expiration: &Vec) { e.storage().instance().set(&EXPIRATION_KEY, expiration) } - -// Store asset index -#[inline] -fn set_asset_index(e: &Env, asset: &Asset, index: u32) { - match asset { - Asset::Stellar(address) => { - e.storage().instance().set(&address, &index); - } - Asset::Other(symbol) => { - e.storage().instance().set(&symbol, &index); - } - } -} diff --git a/oracle/src/mapping.rs b/oracle/src/mapping.rs index d8a1b90..e058e80 100644 --- a/oracle/src/mapping.rs +++ b/oracle/src/mapping.rs @@ -1,57 +1,67 @@ -use soroban_sdk::{Bytes, Env, Vec, U256}; +use soroban_sdk::{Bytes, Vec}; // Each history record occupies 32 bytes in history mask, allowing to store information for up to 256 recent periods const RECORD_SIZE: u32 = 32; +const URECORD_SIZE: usize = 32; +const MAX_HISTORY_SIZE: usize = 256 * URECORD_SIZE; // 256 assets * 32 bytes // Update history records containing a bitmask of all prices recorded within the last update period pub fn update_history_mask( - e: &Env, - mut history_mask: Bytes, + history_mask: Bytes, updates: &Vec, mut updates_delta: u32, ) -> Bytes { - let one = U256::from_u32(e, 1); - //wipe entire history if the gap between updates is too large - if updates_delta > 255 { - history_mask = Bytes::new(e); //start with an empty mask - updates_delta = 1; - } + //create a buffer that can hold the entire history mask + let mut buffer = [0u8; MAX_HISTORY_SIZE]; + let mask_length = history_mask.len() as usize; + if updates_delta < 1 { updates_delta = 1; //this should never happen, but just in case } - //iterate through all updates + if updates_delta > 255 { + //entire history is obsolete - ignore + updates_delta = 1; //reset delta to 1 + } else { + //copy existing history mask into buffer + history_mask.copy_into_slice(&mut buffer[..mask_length]); + } + //iterate through all updates and update corresponding history records in the buffer for (asset_index, price) in updates.iter().enumerate() { - //locate particular asset mask slice position within entire history record - let from = asset_index as u32 * RECORD_SIZE; - let to = from + RECORD_SIZE; - //retrieve previous asset mask - let mut bitmask = if history_mask.len() >= to { - let encoded = history_mask.slice(from..to); - U256::from_be_bytes(e, &encoded) - } else { - U256::from_u32(e, 0) //no previous records for this asset found - }; - //shift existing mask to the left by the number of periods since the last update - //all mask bits older than 256 periods get evicted - bitmask = bitmask.shl(updates_delta); - //set corresponding bit if price found - if price > 0 { - bitmask = bitmask.add(&one); + //position in the mask + let offset = asset_index * URECORD_SIZE; + + //256 bits as two 128 parts + let mut hi = u128::from_be_bytes(buffer[offset..offset + 16].try_into().unwrap()); + let mut lo = u128::from_be_bytes(buffer[offset + 16..offset + 32].try_into().unwrap()); + + if lo > 0 || hi > 0 { + //shift left by the number of skipped periods + (hi, lo) = if updates_delta < 128 { + ( + (hi << updates_delta) | (lo >> (128 - updates_delta)), + lo << updates_delta, + ) + } else { + (lo << (updates_delta & 0x7f), 0) + }; } - //encode into bytes again - let encoded = bitmask.to_be_bytes(); - //write to the history - if history_mask.len() <= from { - //that's new asset, add to the mask - history_mask.append(&encoded); - } else { - //replace bytes - for i in 0..RECORD_SIZE { - history_mask.set(from + i, encoded.get(i).unwrap()); + + //set lowest bit if price found + if price > 0 { + let (new_lo, carry) = lo.overflowing_add(1); + lo = new_lo; + if carry { + (hi, _) = hi.overflowing_add(1); } } + //write back to buffer + buffer[offset..offset + 16].copy_from_slice(&hi.to_be_bytes()); + buffer[offset + 16..offset + 32].copy_from_slice(&lo.to_be_bytes()); } - history_mask //return updated history + + //get total size of updated history mask based on the number of assets and return as Bytes + let updates_length = mask_length.max(updates.len() as usize * URECORD_SIZE); + Bytes::from_slice(history_mask.env(), &buffer[..updates_length]) } // Check whether asset price has been quoted for a certain period based on history records bitmask @@ -61,9 +71,9 @@ pub fn check_history_updated(history_mask: &Bytes, asset_index: u32, period: u32 //and calculate specific bit that we need to check let bit = 1 << (period % 8); //retrieve byte from array - let bytemask = history_mask.get(from).unwrap_or_default(); + let encoded_byte = history_mask.get(from).unwrap_or_default(); //compare with bit mask - bytemask & bit == bit + encoded_byte & bit == bit } // Check whether price update record contains update for given asset by its index diff --git a/oracle/src/price_oracle.rs b/oracle/src/price_oracle.rs index a6dae37..946a171 100644 --- a/oracle/src/price_oracle.rs +++ b/oracle/src/price_oracle.rs @@ -333,7 +333,7 @@ impl PriceOracleContractBase { //prepare and publish update event events::publish_update_event(e, &asset_prices, &all, timestamp); //store new prices - prices::store_prices(e, &update, timestamp, &asset_prices); + prices::store_prices(e, update, timestamp, asset_prices); } // Update contract source code diff --git a/oracle/src/prices.rs b/oracle/src/prices.rs index 8fc7bad..fcc4cd7 100644 --- a/oracle/src/prices.rs +++ b/oracle/src/prices.rs @@ -123,7 +123,7 @@ pub fn update_history_mask(e: &Env, prices: &Vec, timestamp: u64) { } //update the position mask - history_map = mapping::update_history_mask(e, history_map, prices, update_delta as u32); + history_map = mapping::update_history_mask(history_map, prices, update_delta as u32); //store updated timestamps e.storage().instance().set(&HISTORY_KEY, &history_map); } @@ -150,7 +150,7 @@ pub fn load_history_record(e: &Env, timestamp: u64) -> Option { } // Update prices stored in the oracle -pub fn store_prices(e: &Env, update: &PriceUpdate, timestamp: u64, update_v1: &Vec) { +pub fn store_prices(e: &Env, update: PriceUpdate, timestamp: u64, update_v1: Vec) { //validate timestamp let ledger_timestamp = timestamps::ledger_timestamp(&e); let last_timestamp = get_last_timestamp(e); @@ -172,7 +172,7 @@ pub fn store_prices(e: &Env, update: &PriceUpdate, timestamp: u64, update_v1: &V if cache_size > 0 { //if cache size is non-empty, store it in the instance let mut cache = load_price_records_cache(e).unwrap_or(Vec::new(&e)); - cache.push_front((timestamp, update.clone())); + cache.push_front((timestamp, update)); while cache.len() > cache_size { cache.pop_back(); //remove the oldest record if cache size exceeded } @@ -216,6 +216,7 @@ pub fn load_prices(e: &Env, asset_index: u32, records: u32) -> Option lower_boundary { + //TODO: Load `history_map` and `cache` once outside the loop //invoke price fetch callback for each record if let Some(price) = retrieve_asset_price_data(e, asset_index, timestamp) { prices.push_back(price); @@ -240,7 +241,7 @@ fn load_price_records_cache(e: &Env) -> Option> { } // Update price in legacy format (deprecated) -pub fn store_price_v1(e: &Env, updates: &Vec, timestamp: u64, ledgers_to_live: u32) { +pub fn store_price_v1(e: &Env, updates: Vec, timestamp: u64, ledgers_to_live: u32) { //iterate over the updates for (i, price) in updates.iter().enumerate() { //ignore zero prices diff --git a/oracle/src/tests/fetch_prices_tests.rs b/oracle/src/tests/fetch_prices_tests.rs index 228af08..b25343d 100644 --- a/oracle/src/tests/fetch_prices_tests.rs +++ b/oracle/src/tests/fetch_prices_tests.rs @@ -48,7 +48,7 @@ fn store_prices_test( set_ledger_timestamp(&e, 600_000); let mut assets = Vec::new(&e); - for i in 0..10 { + for i in 0..255 { assets.push_back(types::Asset::Other(Symbol::new( &e, &("ASSET_".to_string() + &i.to_string()), @@ -65,9 +65,10 @@ fn store_prices_test( fn set_price(e: &Env, timestamp: u64, assets: &Vec) { let updates = generate_updates(e, &assets, 100); let asset_prices = prices::extract_update_record_prices(e, &updates, assets.len()); + let legacy_update = updates.prices.clone(); //store history timestamps for all assets prices::update_history_mask(e, &asset_prices, timestamp); - prices::store_prices(e, &updates, timestamp, &updates.prices.clone()); + prices::store_prices(e, updates, timestamp, legacy_update); } let mut timestamp = first_timestamp; @@ -93,4 +94,6 @@ fn store_prices_test( expected_first_price_ts / 1000 ); }); + + e.cost_estimate().budget().print(); } diff --git a/oracle/src/tests/prices_tests.rs b/oracle/src/tests/prices_tests.rs index 71c9308..3c33130 100644 --- a/oracle/src/tests/prices_tests.rs +++ b/oracle/src/tests/prices_tests.rs @@ -42,7 +42,10 @@ fn invalid_timestamp_update_test(ts: u64) { &e, types::PriceUpdate { prices: vec![&e, 12345678i128], - mask: generate_update_record_mask(&e, &vec![&e, 12345678i128]), + mask: generate_update_record_mask( + &e, + &std::collections::VecDeque::from([12345678i128]), + ), }, ts, ); @@ -50,7 +53,7 @@ fn invalid_timestamp_update_test(ts: u64) { } #[test] -fn price_update_test() { +fn single_price_update_test() { let e = Env::default(); //register contract to have storage available let contract = e.register_stellar_asset_contract_v2(Address::generate(&e)); @@ -79,7 +82,10 @@ fn price_update_test() { &e, types::PriceUpdate { prices: vec![&e, 12345678i128], - mask: generate_update_record_mask(&e, &vec![&e, 12345678i128]), + mask: generate_update_record_mask( + &e, + &std::collections::VecDeque::from([12345678i128]), + ), }, 900_000, ); diff --git a/oracle/src/tests/util_tests.rs b/oracle/src/tests/util_tests.rs index 1183e56..443aead 100644 --- a/oracle/src/tests/util_tests.rs +++ b/oracle/src/tests/util_tests.rs @@ -44,7 +44,7 @@ fn position_encoding_bitmask_test() { }; updates.push_back(price); } - mask = mapping::update_history_mask(&e, mask, &updates, 1); + mask = mapping::update_history_mask(mask, &updates, 1); } log!(&e, "entire mask", mask); @@ -70,21 +70,21 @@ fn update_record_bitmask_test() { let e = Env::default(); let iterations = 70; - let mut updates = Vec::from_array(&e, [0i128; 254]); + let mut updates = std::collections::VecDeque::from([0i128; 254]); for i in 0..iterations { for asset_index in 0..updates.len() { let price = match i & asset_index == 0 { true => 1, _ => 0, }; - updates.set(asset_index, price); + updates[asset_index] = price; } let mask = generate_update_record_mask(&e, &updates); //log!(&e, "entire mask", mask); for (asset_index, price) in updates.iter().enumerate() { assert_eq!( mapping::check_period_updated(&mask, asset_index as u32), - price > 0 + price > &0 ); } } diff --git a/oracle/src/testutils/generators.rs b/oracle/src/testutils/generators.rs index ebcea25..44f89d0 100644 --- a/oracle/src/testutils/generators.rs +++ b/oracle/src/testutils/generators.rs @@ -8,11 +8,12 @@ use crate::{ }; use alloc::string::ToString; use soroban_sdk::{testutils::Address as _, Address, Bytes, Env, Symbol, Vec}; +use std::collections::VecDeque; -pub fn generate_update_record_mask(e: &Env, updates: &Vec) -> Bytes { +pub fn generate_update_record_mask(e: &Env, updates: &VecDeque) -> Bytes { let mut mask = [0u8; 32]; for (asset, price) in updates.iter().enumerate() { - if price > 0 { + if price > &0 { let (byte, bitmask) = mapping::resolve_period_update_mask_position(asset as u32); let i = byte as usize; let bytemask = mask[i] | bitmask; @@ -38,16 +39,25 @@ pub fn generate_test_env() -> (ConfigData, Env) { (config, env) } -pub fn generate_updates(env: &Env, assets: &Vec, price: i128) -> PriceUpdate { - let mut updates = Vec::new(&env); +pub fn generate_updates( + env: &Env, + assets: &Vec, + price: i128, +) -> (PriceUpdate, VecDeque) { + let mut updates = VecDeque::new(); + let mut filtered_price = Vec::new(&env); for _ in assets.iter() { updates.push_back(price); + filtered_price.push_back(price); } let mask = generate_update_record_mask(env, &updates); - PriceUpdate { - prices: updates, - mask, - } + ( + PriceUpdate { + prices: filtered_price, + mask, + }, + updates, + ) } fn get_random_bool() -> bool { @@ -60,17 +70,38 @@ fn get_random_bool() -> bool { random_bool } -pub fn generate_random_updates(env: &Env, assets: &Vec, price: i128) -> PriceUpdate { - let mut updates = Vec::new(&env); +pub fn generate_random_updates( + env: &Env, + assets: &Vec, + price: i128, +) -> (PriceUpdate, VecDeque) { + let mut updates = VecDeque::new(); + let mut filtered_price = Vec::new(&env); + let mut has_price = false; for _ in assets.iter() { - let price = if get_random_bool() { 0 } else { price }; + //ensure that at least one price is set + let price = if (has_price || updates.len() < assets.len() as usize - 1) && get_random_bool() + { + 0 + } else { + price + }; updates.push_back(price); + if price > 0 { + filtered_price.push_back(price); + } + if price > 0 { + has_price = true; + } } let mask = generate_update_record_mask(env, &updates); - PriceUpdate { - prices: updates, - mask, - } + ( + PriceUpdate { + prices: filtered_price, + mask, + }, + updates, + ) } pub fn generate_assets(e: &Env, count: usize, start_index: u32) -> Vec { diff --git a/pulse-contract/src/tests/contract_admin_tests.rs b/pulse-contract/src/tests/contract_admin_tests.rs index 3f4520c..1f0ca8d 100644 --- a/pulse-contract/src/tests/contract_admin_tests.rs +++ b/pulse-contract/src/tests/contract_admin_tests.rs @@ -55,7 +55,7 @@ fn set_price_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); //build expected event let expected_event = oracle::events::UpdateEvent { @@ -92,7 +92,7 @@ fn set_price_zero_timestamp_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); } #[test] @@ -109,7 +109,7 @@ fn set_price_invalid_timestamp_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); } #[test] @@ -126,7 +126,7 @@ fn set_price_future_timestamp_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); } #[test] @@ -174,8 +174,6 @@ fn asset_update_overflow_test() { env.mock_all_auths(); - env.cost_estimate().budget().reset_unlimited(); - let mut assets = Vec::new(&env); for i in 1..=1000 { assets.push_back(Asset::Other(Symbol::new( @@ -195,15 +193,13 @@ fn price_update_overflow_test() { env.mock_all_auths(); - env.cost_estimate().budget().reset_unlimited(); - - let mut updates = Vec::new(&env); + let mut raw_prices = std::collections::VecDeque::new(); for i in 1..=256 { - updates.push_back(normalize_price(i as i128 + 1)); + raw_prices.push_back(normalize_price(i as i128 + 1)); } - let mask = generate_update_record_mask(&env, &updates); + let mask = generate_update_record_mask(&env, &raw_prices); let update = PriceUpdate { - prices: updates, + prices: Vec::from_iter(&env, raw_prices.into_iter()), mask, }; client.set_price(&update, &600_000); diff --git a/pulse-contract/src/tests/contract_interface_tests.rs b/pulse-contract/src/tests/contract_interface_tests.rs index e89c399..8447d79 100644 --- a/pulse-contract/src/tests/contract_interface_tests.rs +++ b/pulse-contract/src/tests/contract_interface_tests.rs @@ -1,15 +1,17 @@ #![cfg(test)] use oracle::init_contract_with_admin; -use oracle::prices::{self}; use oracle::testutils::{ convert_to_seconds, generate_random_updates, generate_updates, normalize_price, register_token, set_ledger_timestamp, }; use oracle::types::{FeeConfig, PriceData}; -use soroban_sdk::{log, testutils::Address as _, Address, Env, Vec}; +use soroban_sdk::{testutils::Address as _, Address}; use test_case::test_case; +extern crate std; +use std::{collections::VecDeque, println}; + use crate::{PulseOracleContract, PulseOracleContractClient}; #[test] @@ -59,7 +61,7 @@ fn last_timestamp_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); result = client.last_timestamp(); @@ -79,7 +81,7 @@ fn lastprice_test() { env.mock_all_auths(); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); let fee_asset = env .register_stellar_asset_contract_v2(init_data.admin.clone()) @@ -107,29 +109,26 @@ fn prices_update_test(gap: u64, _description: &str) { client.set_cache_size(&3); - let mut history_prices = Vec::new(&env); + let mut history_prices = VecDeque::new(); + println!("setting prices..."); //set more than 256 prices to check that history mask is overwritten correctly for i in 0..(gap + 256) { let timestamp = 600_000 + i * 300_000; if i < 1 || i > gap { let updates = generate_random_updates(&env, &assets, normalize_price(100)); - history_prices.push_front((timestamp, updates.clone())); //set prices for assets - client.set_price(&updates, ×tamp); + client.set_price(&updates.0, ×tamp); + history_prices.push_front((timestamp, Some(updates.1))); } else { //simulate time passage without setting prices to create gaps in updates - let updates = generate_random_updates(&env, &assets, 0); - history_prices.push_front((timestamp, updates.clone())); + history_prices.push_front((timestamp, None)); } set_ledger_timestamp(&env, timestamp / 1000 + 300); } - //prepare an array with zero prices - let mut zero_prices = Vec::new(&env); - for _ in 0..assets.len() { - zero_prices.push_back(0i128); - } + + println!("verifying prices..."); //verify let mut had_gaps = false; @@ -137,32 +136,32 @@ fn prices_update_test(gap: u64, _description: &str) { let mut iterations = 0; for (history_index, (timestamp, updates)) in history_prices.iter().enumerate() { - let all_prices; - if history_index > 255 { - all_prices = zero_prices.clone(); - } else { - let total = assets.len() + 10; //+10 to check that out of range assets are ignored - //get records from generated updates - all_prices = prices::extract_update_record_prices(&env, &updates, total); - } - //match price with mask for each asset in update for (asset_index, asset) in assets.iter().enumerate() { //get oracle-quoted price let oracle_price = client.price(&asset, &(timestamp / 1000)); //get expected price (from generated data) - let expected_price = all_prices.get(asset_index as u32).unwrap_or_default(); - if expected_price > 0 { + let expected_price = match updates { + Some(updates) => { + if history_index > 255 { + &0 + } else { + updates.get(asset_index).unwrap() + } + } + None => &0, + }; + if expected_price > &0 { let price = oracle_price.unwrap_or_else(|| PriceData { price: 0, timestamp: 0, }); assert_eq!( - price.price, expected_price, - "asset {} at timestamp {}", - asset_index, timestamp + price.price, *expected_price, + "asset {} at timestamp {} history index {}", + asset_index, timestamp, history_index ); - assert_eq!(price.timestamp, convert_to_seconds(timestamp)); + assert_eq!(price.timestamp, convert_to_seconds(*timestamp)); had_prices = true; } else { assert!( @@ -178,7 +177,7 @@ fn prices_update_test(gap: u64, _description: &str) { } assert!(had_prices); assert!(had_gaps); - log!(&env, "{} iterations", iterations); + println!("{} iterations", iterations); } #[test]