From 0fbd95cee5cf3042e2e1d1c167926c0d6d2bf73d Mon Sep 17 00:00:00 2001 From: BobZombiE69 <19148762+BobZombiE69@users.noreply.github.com> Date: Sun, 15 Aug 2021 03:36:59 +0430 Subject: [PATCH 01/14] Update sqlite_store.py Fix snapshot crash due to Sqlite query error --- pool/store/sqlite_store.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index 187a5413..203202af 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -218,8 +218,7 @@ async def snapshot_farmer_points(self) -> None: await self.connection.execute( ( "INSERT into points_ss(launcher_id, points, timestamp, delay_time)" - "SELECT launcher_id, points, strftime('%s', 'now'), delay_time from farmer" - "WHERE points != 0" + "SELECT launcher_id, points, strftime('%s', 'now'), delay_time from farmer WHERE points != 0" ) ) From 7f5f0b724f370b964adf50904e2e5fe694c03732 Mon Sep 17 00:00:00 2001 From: BobZombiE69 <19148762+BobZombiE69@users.noreply.github.com> Date: Mon, 16 Aug 2021 14:25:39 +0430 Subject: [PATCH 02/14] Update sqlite_store.py Fix error Databse is locked , when snapshot is running --- pool/store/sqlite_store.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index 203202af..397da75d 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -221,6 +221,9 @@ async def snapshot_farmer_points(self) -> None: "SELECT launcher_id, points, strftime('%s', 'now'), delay_time from farmer WHERE points != 0" ) ) + await cursor.close() + await self.connection.commit() + async def clear_farmer_points(self) -> None: cursor = await self.connection.execute(f"UPDATE farmer set points=0") From 12bc1bfe40c4f2fcc82c55dc3cbacab63fc74b07 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 02:46:50 +0430 Subject: [PATCH 03/14] set sqlite as default db --- pool/payment/payment.py | 10 ++++++---- pool/pool.py | 4 ++-- pool/reward/reward_collector.py | 5 +++-- pool/snapshot/snapshot.py | 5 +++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pool/payment/payment.py b/pool/payment/payment.py index 502356e5..1e1bff15 100644 --- a/pool/payment/payment.py +++ b/pool/payment/payment.py @@ -25,6 +25,7 @@ from ..pay_record import PaymentRecord + class Payment: def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Optional[AbstractPoolStore] = None): self.log = logging @@ -41,7 +42,7 @@ def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Opti self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or PGStore() + self.store: AbstractPoolStore = pool_store or SqlitePoolStore() self.pool_fee = pool_config["pool_fee"] @@ -190,11 +191,13 @@ async def create_payment_loop(self): self.log.info(f"Paying out {mojo_per_point} mojo / point") additions_sub_list: List[Dict] = [ - {"puzzle_hash": self.pool_fee_puzzle_hash, "amount": pool_coin_amount, "launcher_id": self.default_target_puzzle_hash, "points": 0} + {"puzzle_hash": self.pool_fee_puzzle_hash, "amount": pool_coin_amount, + "launcher_id": self.default_target_puzzle_hash, "points": 0} ] for points, ph, launcher in points_and_ph: if points > 0: - additions_sub_list.append({"puzzle_hash": ph, "amount": points * mojo_per_point, "launcher_id": launcher, "points": points}) + additions_sub_list.append( + {"puzzle_hash": ph, "amount": points * mojo_per_point, "launcher_id": launcher, "points": points}) if len(additions_sub_list) == self.max_additions_per_transaction: await self.pending_payments.put(additions_sub_list.copy()) @@ -211,7 +214,6 @@ async def create_payment_loop(self): # Subtract the points from each farmer await self.store.clear_farmer_points() - else: self.log.info(f"No points for any farmer. Waiting {self.payment_interval}") diff --git a/pool/pool.py b/pool/pool.py index 0bdc35b6..967e49f5 100644 --- a/pool/pool.py +++ b/pool/pool.py @@ -49,7 +49,7 @@ from .store.sqlite_store import SqlitePoolStore from .record import FarmerRecord from .util import error_dict, RequestMetadata -from pool.store.pg_store import PGStore + class Pool: def __init__( @@ -77,7 +77,7 @@ def __init__( self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or PGStore() + self.store: AbstractPoolStore = pool_store or SqlitePoolStore() self.pool_fee = pool_config["pool_fee"] diff --git a/pool/reward/reward_collector.py b/pool/reward/reward_collector.py index 70afa22d..b8985fe4 100644 --- a/pool/reward/reward_collector.py +++ b/pool/reward/reward_collector.py @@ -23,10 +23,11 @@ from pool.singleton import create_absorb_transaction, get_singleton_state, get_coin_spend from pool.store.abstract import AbstractPoolStore -from pool.store.pg_store import PGStore +from .store.sqlite_store import SqlitePoolStore from ..reward_record import RewardRecord + class RewardCollector: def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Optional[AbstractPoolStore] = None): self.log = logging @@ -45,7 +46,7 @@ def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Opti self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or PGStore() + self.store: AbstractPoolStore = pool_store or SqlitePoolStore() # This is the wallet fingerprint and ID for the wallet spending the funds from `self.default_target_puzzle_hash` self.wallet_fingerprint = pool_config["wallet_fingerprint"] diff --git a/pool/snapshot/snapshot.py b/pool/snapshot/snapshot.py index a594dd13..401971b1 100644 --- a/pool/snapshot/snapshot.py +++ b/pool/snapshot/snapshot.py @@ -11,7 +11,8 @@ from chia.util.chia_logging import initialize_logging from pool.store.abstract import AbstractPoolStore -from pool.store.pg_store import PGStore +from .store.sqlite_store import SqlitePoolStore + class Snapshot: def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Optional[AbstractPoolStore] = None): @@ -29,7 +30,7 @@ def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Opti self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or PGStore() + self.store: AbstractPoolStore = pool_store or SqlitePoolStore() # Interval for taking snapshot of farmer's points self.snapshot_interval = pool_config["snapshot_interval"] From 1c3a83637475d755d458cd93e3089f0a9bac17df Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:08:02 +0430 Subject: [PATCH 04/14] some fixes --- pool/store/pg_store.py | 16 +++++++++------- pool/store/sqlite_store.py | 39 ++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/pool/store/pg_store.py b/pool/store/pg_store.py index 772aa9cb..d2ae2b4e 100644 --- a/pool/store/pg_store.py +++ b/pool/store/pg_store.py @@ -20,6 +20,7 @@ class PGStore(AbstractPoolStore): """ Pool store based on SQLite. """ + def __init__(self): super().__init__() self.connection: Optional[asyncpg.Pool] = None @@ -75,11 +76,12 @@ async def connect(self): "launcher_id text, /* farmer */" "points bigint, /* farmer's points */" "delay_time bigint, /* delayed time */" - "timestamp bigint, /* snapshot timestamp */ - "ss_type smallint /* 0: normal snapshot, 1: clear snapshot, 2: pool total */ - )" + "timestamp bigint, /* snapshot timestamp */" + "ss_type smallint /* 0: normal snapshot, 1: clear snapshot, 2: pool total */" + ")" ) ) + await self.connection.execute("CREATE INDEX IF NOT EXISTS ss_launcher_id_index on maxi_points_ss(launcher_id)") # create rewards tx table @@ -95,7 +97,7 @@ async def connect(self): ) await self.connection.execute("CREATE INDEX IF NOT EXISTS re_launcher_id_index on maxi_rewards_tx(launcher_id)") - @staticmethod + @ staticmethod def _row_to_farmer_record(row) -> FarmerRecord: return FarmerRecord( bytes.fromhex(row[0]), @@ -133,7 +135,7 @@ async def add_farmer_record(self, farmer_record: FarmerRecord, metadata: Request ) async def get_farmer_record(self, launcher_id: bytes32) -> Optional[FarmerRecord]: - row = await self.connection.fetchrow( + row=await self.connection.fetchrow( "SELECT * from maxi_farmer where launcher_id=$1", launcher_id.hex(), ) @@ -154,9 +156,9 @@ async def update_singleton( is_pool_member: bool, ): if is_pool_member: - entry = (bytes(singleton_tip), bytes(singleton_tip_state), 1, launcher_id.hex()) + entry=(bytes(singleton_tip), bytes(singleton_tip_state), 1, launcher_id.hex()) else: - entry = (bytes(singleton_tip), bytes(singleton_tip_state), 0, launcher_id.hex()) + entry=(bytes(singleton_tip), bytes(singleton_tip_state), 0, launcher_id.hex()) await self.connection.execute( f"UPDATE maxi_farmer SET singleton_tip=$1, singleton_tip_state=$2, is_pool_member=$3 WHERE launcher_id=$4", *entry, diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index b75aacbf..e9a78451 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -14,6 +14,7 @@ from ..pay_record import PaymentRecord from ..reward_record import RewardRecord + class SqlitePoolStore(AbstractPoolStore): """ Pool store based on SQLite. @@ -97,7 +98,6 @@ async def connect(self): await self.connection.commit() - @staticmethod def _row_to_farmer_record(row) -> FarmerRecord: return FarmerRecord( @@ -218,13 +218,20 @@ async def get_farmer_points_and_payout_instructions(self) -> List[Tuple[uint64, async def snapshot_farmer_points(self, ss_type: int) -> None: cursor = await self.connection.execute( f"INSERT into points_ss(launcher_id, points, timestamp, delay_time, ss_type)" - "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer" - "WHERE points != 0", + "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer" + "WHERE points != 0", ss_type ) await cursor.close() await self.connection.commit() + await self.connection.execute( + ( + "INSERT into points_ss (launcher_id, points, timestamp, delay_time, ss_type)" + "SELECT 'pool_total', SUM(points), trunc(extract(epoch from now())), MAX(delay_time), 2 from maxi_farmer" + ) + ) + async def snapshot_farmer_points(self) -> None: cursor = await self.connection.execute( ( @@ -236,7 +243,15 @@ async def snapshot_farmer_points(self) -> None: await cursor.close() await self.connection.commit() - + async def snapshot_pool_points(self) -> None: + cursor = await self.connection.execute( + ( + "INSERT into points_ss(launcher_id, points, timestamp, delay_time,ss_type)" + "SELECT 'pool_total', SUM(points), strftime('%s', 'now'), MAX(delay_time), 2 from farmer" + ) + ) + await cursor.close() + await self.connection.commit() async def clear_farmer_points(self) -> None: cursor = await self.connection.execute( @@ -279,14 +294,14 @@ async def add_payment(self, payment: PaymentRecord): cursor = await self.connection.execute( f"INSERT into payment (launcher_id, amount, payment_type, timestamp, points, txid, note) VALUES(?, ?, ?, ?, ?, ?, ?)", ( - payment.launcher_id.hex(), - payment.payment_amount, - payment.payment_type, - payment.timestamp, - payment.points, - payment.txid, - payment.note - ), + payment.launcher_id.hex(), + payment.payment_amount, + payment.payment_type, + payment.timestamp, + payment.points, + payment.txid, + payment.note + ), ) await cursor.close() await self.connection.commit() From 168f0fd978d4304b2f279b58493475521004d0d6 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:13:13 +0430 Subject: [PATCH 05/14] more fixes --- pool/payment/payment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/payment/payment.py b/pool/payment/payment.py index 1e1bff15..6c169cf6 100644 --- a/pool/payment/payment.py +++ b/pool/payment/payment.py @@ -21,7 +21,7 @@ from chia.wallet.transaction_record import TransactionRecord from pool.store.abstract import AbstractPoolStore -from pool.store.pg_store import PGStore +from .store.sqlite_store import SqlitePoolStore from ..pay_record import PaymentRecord From 8bdd044283d6b326185490b6f1b4c8723c2df6aa Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:20:29 +0430 Subject: [PATCH 06/14] fix --- pool/payment/payment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/payment/payment.py b/pool/payment/payment.py index 6c169cf6..433cb720 100644 --- a/pool/payment/payment.py +++ b/pool/payment/payment.py @@ -21,7 +21,7 @@ from chia.wallet.transaction_record import TransactionRecord from pool.store.abstract import AbstractPoolStore -from .store.sqlite_store import SqlitePoolStore +from pool.store.sqlite_store import SqlitePoolStore from ..pay_record import PaymentRecord From cf227e71a75ad5eeeb5d59e42c871547c8aae45b Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:27:33 +0430 Subject: [PATCH 07/14] fix --- pool/reward/reward_collector.py | 3 ++- pool/snapshot/snapshot.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pool/reward/reward_collector.py b/pool/reward/reward_collector.py index b8985fe4..38c28cf2 100644 --- a/pool/reward/reward_collector.py +++ b/pool/reward/reward_collector.py @@ -23,7 +23,8 @@ from pool.singleton import create_absorb_transaction, get_singleton_state, get_coin_spend from pool.store.abstract import AbstractPoolStore -from .store.sqlite_store import SqlitePoolStore +from pool.store.sqlite_store import SqlitePoolStore + from ..reward_record import RewardRecord diff --git a/pool/snapshot/snapshot.py b/pool/snapshot/snapshot.py index 401971b1..4365304f 100644 --- a/pool/snapshot/snapshot.py +++ b/pool/snapshot/snapshot.py @@ -11,7 +11,7 @@ from chia.util.chia_logging import initialize_logging from pool.store.abstract import AbstractPoolStore -from .store.sqlite_store import SqlitePoolStore +from pool.store.sqlite_store import SqlitePoolStore class Snapshot: @@ -30,7 +30,7 @@ def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Opti self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or SqlitePoolStore() + self.store: AbstractPoolStore = pool_store or PGStore() # Interval for taking snapshot of farmer's points self.snapshot_interval = pool_config["snapshot_interval"] From fcd6b2d08c1268f563bd6126de871e94fde99497 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:36:43 +0430 Subject: [PATCH 08/14] fix --- pool/store/sqlite_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index e9a78451..311dfb7c 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -232,7 +232,7 @@ async def snapshot_farmer_points(self, ss_type: int) -> None: ) ) - async def snapshot_farmer_points(self) -> None: + async def snapshot_farmer_points(self, ss_type: int) -> None: cursor = await self.connection.execute( ( From 54158f7c927ac84ec6a7540699d581a46796db59 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:53:38 +0430 Subject: [PATCH 09/14] fix --- pool/store/sqlite_store.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index 311dfb7c..03d4e3c1 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -225,24 +225,6 @@ async def snapshot_farmer_points(self, ss_type: int) -> None: await cursor.close() await self.connection.commit() - await self.connection.execute( - ( - "INSERT into points_ss (launcher_id, points, timestamp, delay_time, ss_type)" - "SELECT 'pool_total', SUM(points), trunc(extract(epoch from now())), MAX(delay_time), 2 from maxi_farmer" - ) - ) - - async def snapshot_farmer_points(self, ss_type: int) -> None: - cursor = await self.connection.execute( - ( - - "INSERT into points_ss(launcher_id, points, timestamp, delay_time)" - "SELECT launcher_id, points, strftime('%s', 'now'), delay_time from farmer WHERE points != 0" - ) - ) - await cursor.close() - await self.connection.commit() - async def snapshot_pool_points(self) -> None: cursor = await self.connection.execute( ( From 68e5d69feb82c0f617dd2a02c69bcd3eeafe73af Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:56:58 +0430 Subject: [PATCH 10/14] fix --- pool/snapshot/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/snapshot/snapshot.py b/pool/snapshot/snapshot.py index 4365304f..b0d86509 100644 --- a/pool/snapshot/snapshot.py +++ b/pool/snapshot/snapshot.py @@ -30,7 +30,7 @@ def __init__(self, config: Dict, constants: ConsensusConstants, pool_store: Opti self.config = config self.constants = constants - self.store: AbstractPoolStore = pool_store or PGStore() + self.store: AbstractPoolStore = pool_store or SqlitePoolStore() # Interval for taking snapshot of farmer's points self.snapshot_interval = pool_config["snapshot_interval"] From 19b7496d5e384eb4390426f91dd1f822a5b6242b Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 03:59:15 +0430 Subject: [PATCH 11/14] fix --- pool/store/sqlite_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index 03d4e3c1..009a9aaa 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -220,7 +220,7 @@ async def snapshot_farmer_points(self, ss_type: int) -> None: f"INSERT into points_ss(launcher_id, points, timestamp, delay_time, ss_type)" "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer" "WHERE points != 0", - ss_type + ss_type, ) await cursor.close() await self.connection.commit() From eb9126d1d43dc5aee70948f90ddec52f2a322ea5 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 04:18:48 +0430 Subject: [PATCH 12/14] fix --- pool/store/sqlite_store.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index 009a9aaa..a35e0813 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -218,9 +218,8 @@ async def get_farmer_points_and_payout_instructions(self) -> List[Tuple[uint64, async def snapshot_farmer_points(self, ss_type: int) -> None: cursor = await self.connection.execute( f"INSERT into points_ss(launcher_id, points, timestamp, delay_time, ss_type)" - "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer" - "WHERE points != 0", - ss_type, + "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer WHERE points != 0", + ss_type ) await cursor.close() await self.connection.commit() @@ -228,7 +227,7 @@ async def snapshot_farmer_points(self, ss_type: int) -> None: async def snapshot_pool_points(self) -> None: cursor = await self.connection.execute( ( - "INSERT into points_ss(launcher_id, points, timestamp, delay_time,ss_type)" + "INSERT into points_ss(launcher_id, points, timestamp, delay_time, ss_type)" "SELECT 'pool_total', SUM(points), strftime('%s', 'now'), MAX(delay_time), 2 from farmer" ) ) From 86ce76b1498b1aacdebc69a3202768150627c888 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 05:07:31 +0430 Subject: [PATCH 13/14] fix --- pool/store/sqlite_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index a35e0813..a4a9f8b3 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -78,7 +78,7 @@ async def connect(self): "points bigint," "delay_time bigint," "timestamp bigint," - "ss_type smallint)" + "ss_type smallint /* 0: normal snapshot, 1: clear snapshot, 2: pool total */)" ) ) await self.connection.execute("CREATE INDEX IF NOT EXISTS ss_launcher_id_index on points_ss(launcher_id)") From cb48a26e0d143f7c9c5d44c2cb01dc101b738154 Mon Sep 17 00:00:00 2001 From: BobZombiE69 Date: Tue, 24 Aug 2021 05:23:41 +0430 Subject: [PATCH 14/14] fix --- pool/store/sqlite_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/store/sqlite_store.py b/pool/store/sqlite_store.py index a4a9f8b3..5c3706df 100644 --- a/pool/store/sqlite_store.py +++ b/pool/store/sqlite_store.py @@ -219,7 +219,7 @@ async def snapshot_farmer_points(self, ss_type: int) -> None: cursor = await self.connection.execute( f"INSERT into points_ss(launcher_id, points, timestamp, delay_time, ss_type)" "SELECT launcher_id, points, strftime('%s', 'now'), delay_time, ? from farmer WHERE points != 0", - ss_type + (ss_type,) ) await cursor.close() await self.connection.commit()