From c9d281cb287c68a39751ab8aedd3f28982a60a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 17:55:06 +0100 Subject: [PATCH 01/15] feat: allow % to track many addresses --- apps/server/src/api/protocols/users/index.ts | 46 ++++++++++++-------- packages/protocols/src/protocols.ts | 17 ++------ 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/apps/server/src/api/protocols/users/index.ts b/apps/server/src/api/protocols/users/index.ts index 41591bb6..2688b535 100644 --- a/apps/server/src/api/protocols/users/index.ts +++ b/apps/server/src/api/protocols/users/index.ts @@ -46,7 +46,8 @@ export default defineCachedEventHandler(async (event) => { })); return stats; -}, apiCacheConfig); + // }, apiCacheConfig); +}); interface QueryParams { limit: number; @@ -63,22 +64,29 @@ const getProtocolUsersDirect = async ({ } const result = await sql` - SELECT - dapps.id as protocol_name, - COUNT(DISTINCT txs.sender_address) AS unique_senders - FROM - txs - JOIN - dapps ON txs.contract_call_contract_id = ANY (dapps.contracts) - WHERE - txs.type_id = 2 - ${sql.unsafe(dateCondition)} - GROUP BY - dapps.id - ORDER BY - unique_senders DESC - LIMIT ${limit}; - `; +WITH protocol_contracts AS ( + SELECT id, UNNEST(contracts) AS contract_address + FROM dapps +) + +SELECT + protocol_contracts.id as protocol_name, + COUNT(DISTINCT txs.sender_address) AS unique_senders +FROM + txs +JOIN + protocol_contracts ON txs.contract_call_contract_id LIKE protocol_contracts.contract_address +WHERE + txs.type_id = 2 + ${sql.unsafe(dateCondition)} +GROUP BY + protocol_contracts.id +ORDER BY + unique_senders DESC +LIMIT ${limit}; + `; + + console.log("result", JSON.stringify(result, null, 2)); return result; }; @@ -125,7 +133,7 @@ address_txs AS ( SELECT tx_id, index_block_hash, microblock_hash, recipient FROM nft_events ) sub - JOIN protocol_contracts ON sub.address = protocol_contracts.contract_address + JOIN protocol_contracts ON sub.address LIKE protocol_contracts.contract_address ) SELECT @@ -147,5 +155,7 @@ ORDER BY LIMIT ${limit}; `; + console.log("result", JSON.stringify(result, null, 2)); + return result; }; diff --git a/packages/protocols/src/protocols.ts b/packages/protocols/src/protocols.ts index be13dda4..bcca51ba 100644 --- a/packages/protocols/src/protocols.ts +++ b/packages/protocols/src/protocols.ts @@ -124,6 +124,7 @@ export const protocolsInfo: { contracts: [ "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v1", "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v2", + "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v3", ], }, stackswap: { @@ -144,21 +145,9 @@ export const protocolsInfo: { "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-path2", "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-core", // V1 - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0001", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0002", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0003", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0004", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0005", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0006", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0007", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0008", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0009", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0010", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0011", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0012", - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-v1_0_0-0013", + "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.univ2-pool-%", // Stables - "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.curve-pool-v1_1_0-0001", + "SP1Y5YSTAHZ88XYK1VPDH24GY0HPX5J4JECTMY4A1.curve-pool-%", ], }, zest: { From 5fdb5b6f2c59555b1fb370941063c2bc43a7deb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 17:58:50 +0100 Subject: [PATCH 02/15] update --- packages/protocols/src/protocols.ts | 63 ++++------------------------- 1 file changed, 7 insertions(+), 56 deletions(-) diff --git a/packages/protocols/src/protocols.ts b/packages/protocols/src/protocols.ts index bcca51ba..06a9f72b 100644 --- a/packages/protocols/src/protocols.ts +++ b/packages/protocols/src/protocols.ts @@ -35,9 +35,7 @@ export const protocolsInfo: { "SP102V8P0F7JX67ARQ77WEA3D3CFB5XW39REDT0AM.amm-pool-v2-01", "SP102V8P0F7JX67ARQ77WEA3D3CFB5XW39REDT0AM.amm-vault-v2-01", "SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.amm-swap-pool-v1-1", - "SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.swap-helper-v1-01", - "SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.swap-helper-v1-02", - "SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.swap-helper-v1-03", + "SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.swap-helper-%", ], }, arkadiko: { @@ -47,7 +45,7 @@ export const protocolsInfo: { website: "https://arkadiko.finance/", x: "https://twitter.com/ArkadikoFinance", contracts: [ - "SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-v2-1", + "SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-%", "SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-multi-hop-swap-v1-1", ], }, @@ -59,56 +57,13 @@ export const protocolsInfo: { // List of contract https://docs.bitflow.finance/bitflow-documentation/developers/deployed-contracts/stacks contracts: [ // StableSwap Contracts - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-stx-ststx-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-stx-ststx-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-usda-susdt-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-aeusdc-susdt-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-usda-aeusdc-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-usda-aeusdc-v-1-4", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-abtc-xbtc-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-abtc-xbtc-v-1-2", - "SM1793C4R5PZ4NS4VQ4WMP7SKKYVH8JZEWSZ9HCCR.stableswap-pool-aeusdc-usdh-v-1-1", - "SM1793C4R5PZ4NS4VQ4WMP7SKKYVH8JZEWSZ9HCCR.stableswap-pool-aeusdc-usdh-v-1-2", + "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-%", // Earn Contracts - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-stx-ststx-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-stx-ststx-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-usda-susdt-v-1-3", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-aeusdc-susdt-v-1-3", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-usda-aeusdc-v-1-3", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-usda-aeusdc-v-1-5", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-abtc-xbtc-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-abtc-xbtc-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-abtc-xbtc-v-1-3", + "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-%", // Router Contracts - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-arkadiko-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-velar-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-velar-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-alex-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-alex-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-alex-v-2-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-ststx-bitflow-xyk-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-usda-arkadiko-alex-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stx-usda-arkadiko-velar-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-xyk-arkadiko-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-xyk-velar-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-xyk-alex-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-xyk-alex-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-velar-alex-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-velar-alex-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-aeusdc-ststx-v-2-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-susdt-ststx-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-xyk-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-xyk-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-velar-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-velar-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-alex-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-alex-v-1-2", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-stableswap-usda-aeusdc-v-1-1", + "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.router-%", // Wrapper Contracts - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.wrapper-velar-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.wrapper-alex-v-2-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.wrapper-velar-multihop-v-1-1", - "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.wrapper-arkadiko-v-1-1", + "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.wrapper-%", // Core Contracts "SM1793C4R5PZ4NS4VQ4WMP7SKKYVH8JZEWSZ9HCCR.xyk-core-v-1-1", "SM1793C4R5PZ4NS4VQ4WMP7SKKYVH8JZEWSZ9HCCR.stableswap-core-v-1-1", @@ -121,11 +76,7 @@ export const protocolsInfo: { "Liquid stacking on Stacks. Earn 10% APY on STX with instant liquidity 📈", website: "https://www.stackingdao.com/", x: "https://twitter.com/StackingDao", - contracts: [ - "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v1", - "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v2", - "SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-v3", - ], + contracts: ["SP4SZE494VC2YC5JYG7AYFQ44F5Q4PYV7DVMDPBG.stacking-dao-core-%"], }, stackswap: { name: "StackSwap", From 4ff68d1d6c446fae9421c9fe34baa52ee9d2f373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 17:59:45 +0100 Subject: [PATCH 03/15] Update protocols.ts --- packages/protocols/src/protocols.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/protocols/src/protocols.ts b/packages/protocols/src/protocols.ts index 06a9f72b..87ac292f 100644 --- a/packages/protocols/src/protocols.ts +++ b/packages/protocols/src/protocols.ts @@ -58,6 +58,7 @@ export const protocolsInfo: { contracts: [ // StableSwap Contracts "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.stableswap-%", + "SM1793C4R5PZ4NS4VQ4WMP7SKKYVH8JZEWSZ9HCCR.stableswap-%", // Earn Contracts "SPQC38PW542EQJ5M11CR25P7BS1CA6QT4TBXGB3M.earn-%", // Router Contracts From 03345ed3c202d142efa3e06bbcde13f0caf4a41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 18:19:53 +0100 Subject: [PATCH 04/15] logic --- .../src/api/transactions/unique-senders/index.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/server/src/api/transactions/unique-senders/index.ts b/apps/server/src/api/transactions/unique-senders/index.ts index a59caf9c..eb867c24 100644 --- a/apps/server/src/api/transactions/unique-senders/index.ts +++ b/apps/server/src/api/transactions/unique-senders/index.ts @@ -64,7 +64,11 @@ address_txs AS ( SELECT tx_id, index_block_hash, microblock_hash, recipient FROM nft_events ) sub - WHERE address IN (SELECT contract_address FROM protocol_contracts) + WHERE EXISTS ( + SELECT 1 + FROM protocol_contracts + WHERE sub.address LIKE protocol_contracts.contract_address + ) ) SELECT @@ -82,6 +86,8 @@ ORDER BY mb.month ASC `; + console.log("result", JSON.stringify(result, null, 2)); + const stats: TransactionUniqueSendersRouteResponse = result.map((row) => ({ // format of the month is "2021-08-01 00:00:00+00" // we want to output "2021-08" @@ -90,4 +96,5 @@ ORDER BY })); return stats; -}, apiCacheConfig); + // }, apiCacheConfig); +}); From c358c5960ffe03569eb173390606b7835ce07f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 18:39:13 +0100 Subject: [PATCH 05/15] get it working --- .../api/transactions/unique-senders/index.ts | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/server/src/api/transactions/unique-senders/index.ts b/apps/server/src/api/transactions/unique-senders/index.ts index eb867c24..73024433 100644 --- a/apps/server/src/api/transactions/unique-senders/index.ts +++ b/apps/server/src/api/transactions/unique-senders/index.ts @@ -40,35 +40,34 @@ protocol_contracts AS ( address_txs AS ( SELECT DISTINCT tx_id, index_block_hash, microblock_hash FROM ( - SELECT tx_id, index_block_hash, microblock_hash, contract_call_contract_id AS address - FROM txs - UNION ALL SELECT tx_id, index_block_hash, microblock_hash, principal FROM principal_stx_txs + WHERE principal LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM stx_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM stx_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM ft_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM ft_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM nft_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM nft_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) ) sub - WHERE EXISTS ( - SELECT 1 - FROM protocol_contracts - WHERE sub.address LIKE protocol_contracts.contract_address - ) ) SELECT @@ -86,8 +85,6 @@ ORDER BY mb.month ASC `; - console.log("result", JSON.stringify(result, null, 2)); - const stats: TransactionUniqueSendersRouteResponse = result.map((row) => ({ // format of the month is "2021-08-01 00:00:00+00" // we want to output "2021-08" @@ -96,5 +93,4 @@ ORDER BY })); return stats; - // }, apiCacheConfig); -}); +}, apiCacheConfig); From f7a8837ec0e5ef4d645dcbaa237eab95496ac96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 18:47:12 +0100 Subject: [PATCH 06/15] Update index.get.ts --- apps/server/src/api/transactions/index.get.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/server/src/api/transactions/index.get.ts b/apps/server/src/api/transactions/index.get.ts index fc4508d2..e382c9f2 100644 --- a/apps/server/src/api/transactions/index.get.ts +++ b/apps/server/src/api/transactions/index.get.ts @@ -17,9 +17,9 @@ type TransactionsRouteResponse = (ContractCallTransaction & { export default defineCachedEventHandler(async (event) => { const query = await getValidatedQueryZod(event, transactionsRouteSchema); - let protocolCondition = ""; + let protocolContractsCondition = ""; if (query.protocol) { - protocolCondition = `AND dapps.id = '${query.protocol}'`; + protocolContractsCondition = `WHERE id = '${query.protocol}'`; } const result = await sql< @@ -28,16 +28,21 @@ export default defineCachedEventHandler(async (event) => { tx_id: Buffer; }[] >` +WITH protocol_contracts AS ( + SELECT id, UNNEST(contracts) AS contract_address + FROM dapps + ${sql.unsafe(protocolContractsCondition)} +) + SELECT - dapps.id as protocol, + protocol_contracts.id as protocol, tx_id FROM txs JOIN - dapps ON txs.contract_call_contract_id = ANY (dapps.contracts) + protocol_contracts ON txs.contract_call_contract_id LIKE protocol_contracts.contract_address WHERE txs.type_id = 2 - ${sql.unsafe(protocolCondition)} AND canonical = TRUE AND microblock_canonical = TRUE ORDER BY From 04c06591d3ac0aa6d0cdbbf417d82e9ae71d0161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 18:51:46 +0100 Subject: [PATCH 07/15] tmp --- apps/server/src/api/transactions/stats/index.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index c2e2f2fd..526896e6 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -31,31 +31,34 @@ WITH protocol_contracts AS ( address_txs AS ( SELECT DISTINCT tx_id, index_block_hash, microblock_hash FROM ( - SELECT tx_id, index_block_hash, microblock_hash, contract_call_contract_id AS address - FROM txs - UNION ALL SELECT tx_id, index_block_hash, microblock_hash, principal FROM principal_stx_txs + WHERE principal LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM stx_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM stx_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM ft_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM ft_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM nft_events + WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM nft_events + WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) ) sub - WHERE address IN (SELECT contract_address FROM protocol_contracts) ) SELECT @@ -69,10 +72,13 @@ JOIN AND atxs.microblock_hash = txs.microblock_hash `; + console.log("Result", result); + const stats: TransactionStatsRouteResponse = { count: Number.parseInt(result[0].count), unique_senders: Number.parseInt(result[0].unique_senders), }; return stats; -}, apiCacheConfig); + // }, apiCacheConfig); +}); From cbc38d8c6f6e2b9ec45c2f2317b1fc9e30c2b06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:12:05 +0100 Subject: [PATCH 08/15] try --- .../src/api/transactions/stats/index.ts | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index 526896e6..f55a1003 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -16,6 +16,8 @@ type TransactionStatsRouteResponse = { export default defineCachedEventHandler(async (event) => { const query = await getValidatedQueryZod(event, transactionStatsRouteSchema); + const startTime = Date.now(); + let protocolContractsCondition = ""; if (query.protocol) { protocolContractsCondition = `WHERE dapps.id = '${query.protocol}'`; @@ -29,47 +31,40 @@ WITH protocol_contracts AS ( ), address_txs AS ( - SELECT DISTINCT tx_id, index_block_hash, microblock_hash - FROM ( - SELECT tx_id, index_block_hash, microblock_hash, principal + ( + SELECT tx_id, index_block_hash, microblock_hash FROM principal_stx_txs WHERE principal LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, sender + ) + UNION + ( + SELECT tx_id, index_block_hash, microblock_hash FROM stx_events WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, recipient - FROM stx_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, sender + OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + ) + UNION + ( + SELECT tx_id, index_block_hash, microblock_hash FROM ft_events WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, recipient - FROM ft_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, sender + OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + ) + UNION + ( + SELECT tx_id, index_block_hash, microblock_hash FROM nft_events WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - UNION ALL - SELECT tx_id, index_block_hash, microblock_hash, recipient - FROM nft_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - ) sub + OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + ) ) SELECT - COUNT(DISTINCT atxs.tx_id) AS count, + COUNT(DISTINCT address_txs.tx_id) AS count, COUNT(DISTINCT txs.sender_address) AS unique_senders -FROM - txs -JOIN - address_txs atxs ON atxs.tx_id = txs.tx_id - AND atxs.index_block_hash = txs.index_block_hash - AND atxs.microblock_hash = txs.microblock_hash + FROM address_txs + INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash) + WHERE canonical = TRUE AND microblock_canonical = TRUE `; console.log("Result", result); @@ -79,6 +74,9 @@ JOIN unique_senders: Number.parseInt(result[0].unique_senders), }; + const endTime = Date.now(); + console.log(`Transaction stats took ${endTime - startTime}ms`); + return stats; // }, apiCacheConfig); }); From 12425af23f2effb32d6610cef97c56693fbe4ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:17:12 +0100 Subject: [PATCH 09/15] improve perf --- .../src/api/transactions/stats/index.ts | 61 ++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index f55a1003..77a8ff42 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -30,41 +30,60 @@ WITH protocol_contracts AS ( ${sql.unsafe(protocolContractsCondition)} ), +-- Materialize protocol_contracts to avoid multiple scans +protocol_contracts_mat AS MATERIALIZED ( + SELECT contract_address FROM protocol_contracts +), + address_txs AS ( - ( + SELECT DISTINCT tx_id, index_block_hash, microblock_hash + FROM ( SELECT tx_id, index_block_hash, microblock_hash FROM principal_stx_txs - WHERE principal LIKE ANY (SELECT contract_address FROM protocol_contracts) - ) - UNION - ( + WHERE EXISTS ( + SELECT 1 FROM protocol_contracts_mat + WHERE principal_stx_txs.principal LIKE protocol_contracts_mat.contract_address + ) + + UNION ALL + SELECT tx_id, index_block_hash, microblock_hash FROM stx_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - ) - UNION - ( + WHERE EXISTS ( + SELECT 1 FROM protocol_contracts_mat + WHERE stx_events.sender LIKE protocol_contracts_mat.contract_address + OR stx_events.recipient LIKE protocol_contracts_mat.contract_address + ) + + UNION ALL + SELECT tx_id, index_block_hash, microblock_hash FROM ft_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - ) - UNION - ( + WHERE EXISTS ( + SELECT 1 FROM protocol_contracts_mat + WHERE ft_events.sender LIKE protocol_contracts_mat.contract_address + OR ft_events.recipient LIKE protocol_contracts_mat.contract_address + ) + + UNION ALL + SELECT tx_id, index_block_hash, microblock_hash FROM nft_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) - OR recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) - ) + WHERE EXISTS ( + SELECT 1 FROM protocol_contracts_mat + WHERE nft_events.sender LIKE protocol_contracts_mat.contract_address + OR nft_events.recipient LIKE protocol_contracts_mat.contract_address + ) + ) combined_events ) SELECT COUNT(DISTINCT address_txs.tx_id) AS count, COUNT(DISTINCT txs.sender_address) AS unique_senders - FROM address_txs - INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash) - WHERE canonical = TRUE AND microblock_canonical = TRUE +FROM address_txs +INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash) +WHERE txs.canonical = TRUE +AND txs.microblock_canonical = TRUE; `; console.log("Result", result); From 676c015e7788761d1259cb3d4237d4cedcdfb7c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:21:21 +0100 Subject: [PATCH 10/15] simplify --- .../src/api/transactions/stats/index.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index 77a8ff42..3abcc7fd 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -30,19 +30,14 @@ WITH protocol_contracts AS ( ${sql.unsafe(protocolContractsCondition)} ), --- Materialize protocol_contracts to avoid multiple scans -protocol_contracts_mat AS MATERIALIZED ( - SELECT contract_address FROM protocol_contracts -), - address_txs AS ( SELECT DISTINCT tx_id, index_block_hash, microblock_hash FROM ( SELECT tx_id, index_block_hash, microblock_hash FROM principal_stx_txs WHERE EXISTS ( - SELECT 1 FROM protocol_contracts_mat - WHERE principal_stx_txs.principal LIKE protocol_contracts_mat.contract_address + SELECT 1 FROM protocol_contracts + WHERE principal_stx_txs.principal LIKE protocol_contracts.contract_address ) UNION ALL @@ -50,9 +45,9 @@ address_txs AS ( SELECT tx_id, index_block_hash, microblock_hash FROM stx_events WHERE EXISTS ( - SELECT 1 FROM protocol_contracts_mat - WHERE stx_events.sender LIKE protocol_contracts_mat.contract_address - OR stx_events.recipient LIKE protocol_contracts_mat.contract_address + SELECT 1 FROM protocol_contracts + WHERE stx_events.sender LIKE protocol_contracts.contract_address + OR stx_events.recipient LIKE protocol_contracts.contract_address ) UNION ALL @@ -60,9 +55,9 @@ address_txs AS ( SELECT tx_id, index_block_hash, microblock_hash FROM ft_events WHERE EXISTS ( - SELECT 1 FROM protocol_contracts_mat - WHERE ft_events.sender LIKE protocol_contracts_mat.contract_address - OR ft_events.recipient LIKE protocol_contracts_mat.contract_address + SELECT 1 FROM protocol_contracts + WHERE ft_events.sender LIKE protocol_contracts.contract_address + OR ft_events.recipient LIKE protocol_contracts.contract_address ) UNION ALL @@ -70,9 +65,9 @@ address_txs AS ( SELECT tx_id, index_block_hash, microblock_hash FROM nft_events WHERE EXISTS ( - SELECT 1 FROM protocol_contracts_mat - WHERE nft_events.sender LIKE protocol_contracts_mat.contract_address - OR nft_events.recipient LIKE protocol_contracts_mat.contract_address + SELECT 1 FROM protocol_contracts + WHERE nft_events.sender LIKE protocol_contracts.contract_address + OR nft_events.recipient LIKE protocol_contracts.contract_address ) ) combined_events ) From d9e9cda3ddd2fe5ec6b6637fd51035e375909f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:24:00 +0100 Subject: [PATCH 11/15] Update index.ts --- apps/server/src/api/transactions/stats/index.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index 3abcc7fd..c05df119 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -16,8 +16,6 @@ type TransactionStatsRouteResponse = { export default defineCachedEventHandler(async (event) => { const query = await getValidatedQueryZod(event, transactionStatsRouteSchema); - const startTime = Date.now(); - let protocolContractsCondition = ""; if (query.protocol) { protocolContractsCondition = `WHERE dapps.id = '${query.protocol}'`; @@ -80,7 +78,6 @@ INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash) WHERE txs.canonical = TRUE AND txs.microblock_canonical = TRUE; `; - console.log("Result", result); const stats: TransactionStatsRouteResponse = { @@ -88,9 +85,5 @@ AND txs.microblock_canonical = TRUE; unique_senders: Number.parseInt(result[0].unique_senders), }; - const endTime = Date.now(); - console.log(`Transaction stats took ${endTime - startTime}ms`); - return stats; - // }, apiCacheConfig); -}); +}, apiCacheConfig); From 51452c1beaf4f5c2ea63ada2e6daa4ee6b0acb62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:32:17 +0100 Subject: [PATCH 12/15] fix --- .../src/api/transactions/stats/index.ts | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index c05df119..df859ca9 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -23,9 +23,13 @@ export default defineCachedEventHandler(async (event) => { const result = await sql` WITH protocol_contracts AS ( - SELECT UNNEST(contracts) AS contract_address - FROM dapps - ${sql.unsafe(protocolContractsCondition)} + SELECT DISTINCT contract_id as contract_address + FROM smart_contracts + WHERE contract_id LIKE ANY ( + SELECT contract_address + FROM dapps, UNNEST(contracts) AS contract_address + ${sql.unsafe(protocolContractsCondition)} + ) ), address_txs AS ( @@ -33,40 +37,28 @@ address_txs AS ( FROM ( SELECT tx_id, index_block_hash, microblock_hash FROM principal_stx_txs - WHERE EXISTS ( - SELECT 1 FROM protocol_contracts - WHERE principal_stx_txs.principal LIKE protocol_contracts.contract_address - ) + WHERE principal IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash FROM stx_events - WHERE EXISTS ( - SELECT 1 FROM protocol_contracts - WHERE stx_events.sender LIKE protocol_contracts.contract_address - OR stx_events.recipient LIKE protocol_contracts.contract_address - ) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) + OR recipient IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash FROM ft_events - WHERE EXISTS ( - SELECT 1 FROM protocol_contracts - WHERE ft_events.sender LIKE protocol_contracts.contract_address - OR ft_events.recipient LIKE protocol_contracts.contract_address - ) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) + OR recipient IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash FROM nft_events - WHERE EXISTS ( - SELECT 1 FROM protocol_contracts - WHERE nft_events.sender LIKE protocol_contracts.contract_address - OR nft_events.recipient LIKE protocol_contracts.contract_address - ) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) + OR recipient IN (SELECT contract_address FROM protocol_contracts) ) combined_events ) From 219b5335f3c892130519e1025caaf5dbfacc43f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:32:50 +0100 Subject: [PATCH 13/15] Update index.ts --- apps/server/src/api/transactions/stats/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/api/transactions/stats/index.ts b/apps/server/src/api/transactions/stats/index.ts index df859ca9..b0c4982f 100644 --- a/apps/server/src/api/transactions/stats/index.ts +++ b/apps/server/src/api/transactions/stats/index.ts @@ -70,7 +70,6 @@ INNER JOIN txs USING (tx_id, index_block_hash, microblock_hash) WHERE txs.canonical = TRUE AND txs.microblock_canonical = TRUE; `; - console.log("Result", result); const stats: TransactionStatsRouteResponse = { count: Number.parseInt(result[0].count), From b54a1ecd3093ce765d026b3cd97ad035bfe688f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:39:46 +0100 Subject: [PATCH 14/15] better --- .../api/transactions/unique-senders/index.ts | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/apps/server/src/api/transactions/unique-senders/index.ts b/apps/server/src/api/transactions/unique-senders/index.ts index 73024433..127587d4 100644 --- a/apps/server/src/api/transactions/unique-senders/index.ts +++ b/apps/server/src/api/transactions/unique-senders/index.ts @@ -19,6 +19,8 @@ export default defineCachedEventHandler(async (event) => { transactionUniqueSendersRouteSchema, ); + const startTime = Date.now(); + const result = await sql` WITH monthly_blocks AS ( SELECT @@ -32,9 +34,13 @@ WITH monthly_blocks AS ( ), protocol_contracts AS ( - SELECT UNNEST(contracts) AS contract_address - FROM dapps - WHERE id = ${query.protocol} + SELECT DISTINCT contract_id as contract_address + FROM smart_contracts + WHERE contract_id LIKE ANY ( + SELECT contract_address + FROM dapps, UNNEST(contracts) AS contract_address + WHERE id = ${query.protocol} + ) ), address_txs AS ( @@ -42,31 +48,31 @@ address_txs AS ( FROM ( SELECT tx_id, index_block_hash, microblock_hash, principal FROM principal_stx_txs - WHERE principal LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE principal IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM stx_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM stx_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE recipient IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM ft_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM ft_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE recipient IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, sender FROM nft_events - WHERE sender LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE sender IN (SELECT contract_address FROM protocol_contracts) UNION ALL SELECT tx_id, index_block_hash, microblock_hash, recipient FROM nft_events - WHERE recipient LIKE ANY (SELECT contract_address FROM protocol_contracts) + WHERE recipient IN (SELECT contract_address FROM protocol_contracts) ) sub ) @@ -85,6 +91,13 @@ ORDER BY mb.month ASC `; + const endTime = Date.now(); + console.log( + `[api/transactions/unique-senders] ${query.protocol} took ${ + endTime - startTime + }ms`, + ); + const stats: TransactionUniqueSendersRouteResponse = result.map((row) => ({ // format of the month is "2021-08-01 00:00:00+00" // we want to output "2021-08" From dba5a19d660f28e3925d1840b901340030e16835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Pradel?= Date: Fri, 29 Nov 2024 19:46:00 +0100 Subject: [PATCH 15/15] Update index.ts --- apps/server/src/api/transactions/unique-senders/index.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/server/src/api/transactions/unique-senders/index.ts b/apps/server/src/api/transactions/unique-senders/index.ts index 127587d4..f7a45ee1 100644 --- a/apps/server/src/api/transactions/unique-senders/index.ts +++ b/apps/server/src/api/transactions/unique-senders/index.ts @@ -19,8 +19,6 @@ export default defineCachedEventHandler(async (event) => { transactionUniqueSendersRouteSchema, ); - const startTime = Date.now(); - const result = await sql` WITH monthly_blocks AS ( SELECT @@ -91,13 +89,6 @@ ORDER BY mb.month ASC `; - const endTime = Date.now(); - console.log( - `[api/transactions/unique-senders] ${query.protocol} took ${ - endTime - startTime - }ms`, - ); - const stats: TransactionUniqueSendersRouteResponse = result.map((row) => ({ // format of the month is "2021-08-01 00:00:00+00" // we want to output "2021-08"