[consensus] Enable Urkel Tree compaction

CHANGELOG.md

@@ -2,11 +2,21 @@
 
 ## unreleased
 
+**When upgrading to this version of hsd you must pass
+`--chain-migrate=3` when you run it for the first time.**
+
 ### Node changes
-  - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string
+ - `FullNode` and `SPVNode` now accept the option `--agent` which adds a string
   to the user-agent of the node (which will already contain hsd version) and is
-  sent to peers in the version packet. Strings must not contain slashes and total
-  user-agent string must be 255 characters or less.
+  sent to peers in the version packet. Strings must not contain slashes and
+  total user-agent string must be less than 255 characters.
+
+  - `FullNode` parses new configuration option `--compact-tree-on-init` and
+  `--compact-tree-init-interval` which will compact the Urkel Tree when the node
+  first opens, by deleting historical data. It will try to compact it again
+  after `tree-init-interval` has passed. Compaction will keep up to the last 288
+  blocks worth of tree data on disk (7-8 tree intervals) exposing the node to a
+  similar deep reorganization vulnerability as a chain-pruning node.
 
 ## v3.0.0
 
@@ -26,8 +36,10 @@
 ### Wallet API changes
 
 - New RPC methods:
-  - `signmessagewithname`: Like `signmessage` but uses a name instead of an address. The owner's address will be used to sign the message.
-  - `verifymessagewithname`: Like `verifymessage` but uses a name instead of an address. The owner's address will be used to verify the message.
+  - `signmessagewithname`: Like `signmessage` but uses a name instead of an
+    address. The owner's address will be used to sign the message.
+  - `verifymessagewithname`: Like `verifymessage` but uses a name instead of an
+    address. The owner's address will be used to verify the message.
 
 - New wallet creation accepts parameter `language` to generate the mnemonic phrase.
 

lib/blockchain/chain.js

@@ -100,8 +100,12 @@ class Chain extends AsyncEmitter {
 
     this.setDeploymentState(state);
 
-    if (!this.options.spv)
-      await this.syncTree();
+    if (!this.options.spv) {
+      const sync = await this.tryCompact();
+
+      if (sync)
+        await this.syncTree();
+    }
 
     this.logger.memory();
 
@@ -121,17 +125,71 @@ class Chain extends AsyncEmitter {
     return this.db.close();
   }
 
+  /**
+   * Check if we need to compact tree data.
+   * @returns {Promise<Boolean>} - Should we sync
+   */
+
+  async tryCompact() {
+    if (this.options.spv)
+      return false;
+
+    if (!this.options.compactTreeOnInit)
+      return true;
+
+    const {txStart} = this.network;
+    const {keepBlocks} = this.network.block;
+    const startFrom = txStart + keepBlocks;
+
+    if (this.height <= startFrom)
+      return true;
+
+    const {compactionHeight} = await this.db.getTreeState();
+    const {compactTreeInitInterval} = this.options;
+    const compactFrom = compactionHeight + keepBlocks + compactTreeInitInterval;
+
+    if (compactFrom > this.height) {
+      this.logger.debug(
+        `Tree will compact when restarted after height ${compactFrom}.`);
+      return true;
+    }
+
+    // Compact tree calls syncTree so we don't want to rerun it.
+    await this.compactTree();
+    return false;
+  }
+
   /**
    * Sync tree state.
    */
 
   async syncTree() {
-    const {treeInterval} = this.network.names;
-    const last = this.height - (this.height % treeInterval);
-
     this.logger.info('Synchronizing Tree with block history...');
 
-    for (let height = last + 1; height <= this.height; height++) {
+    // Current state of the tree, loaded from chain database and
+    // injected in chainDB.open(). It should be in the most
+    // recently-committed state, which should have been at the last
+    // tree interval. We might also need to recover from a
+    // failed compactTree() operation. Either way, there might have been
+    // new blocks added to the chain since then.
+    const currentRoot = this.db.treeRoot();
+
+    // We store commit height for the tree in the tree state.
+    // commitHeight is the height of the block that committed tree root.
+    // Note that the block at commitHeight has different tree root.
+    const treeState = await this.db.getTreeState();
+    const {commitHeight} = treeState;
+
+    // sanity check
+    if (commitHeight < this.height) {
+      const entry = await this.db.getEntryByHeight(commitHeight + 1);
+      assert(entry.treeRoot.equals(treeState.treeRoot));
+      assert(entry.treeRoot.equals(currentRoot));
+    }
+
+    // Replay all blocks since the last tree interval to rebuild
+    // the `txn` which is the in-memory delta between tree interval commitments.
+    for (let height = commitHeight + 1; height <= this.height; height++) {
       const entry = await this.db.getEntryByHeight(height);
       assert(entry);
 
@@ -147,8 +205,8 @@ class Chain extends AsyncEmitter {
       for (const tx of block.txs)
         await this.verifyCovenants(tx, view, height, hardened);
 
-      assert((height % this.network.names.treeInterval) !== 0);
-
+      // If the chain replay crosses a tree interval, it will commit
+      // and write to disk in saveNames(), resetting the `txn` like usual.
       await this.db.saveNames(view, entry, false);
     }
 
@@ -2044,6 +2102,98 @@ class Chain extends AsyncEmitter {
     }
   }
 
+  /**
+   * Compact the Urkel Tree.
+   * Removes all historical state and all data not
+   * linked directly to the provided root node hash.
+   * @returns {Promise}
+   */
+
+  async compactTree() {
+    if (this.options.spv)
+      return;
+
+    if (this.height < this.network.block.keepBlocks)
+      throw new Error('Chain is too short to compact tree.');
+
+    const unlock = await this.locker.lock();
+    this.logger.info('Compacting Urkel Tree...');
+
+    // To support chain reorgs of limited depth we compact the tree
+    // to some commitment point in recent history, then rebuild it from there
+    // back up to the current chain tip. In order to support pruning nodes,
+    // all blocks above this depth must be available on disk.
+    // This actually further reduces the ability for a pruning node to recover
+    // from a deep reorg. On mainnet, `keepBlocks` is 288. A normal pruning
+    // node can recover from a reorg up to that depth. Compacting the tree
+    // potentially reduces that depth to 288 - 36 = 252. A reorg deeper than
+    // that will result in a `MissingNodeError` thrown by Urkel inside
+    // chain.saveNames() as it tries to restore a deleted state.
+
+    // Oldest block available to a pruning node.
+    const oldestBlock = this.height - this.network.block.keepBlocks;
+
+    const {treeInterval} = this.network.names;
+
+    // Distance from that block to the start of the oldest tree interval.
+    const toNextInterval = (treeInterval - (oldestBlock % treeInterval))
+      % treeInterval;
+
+    // Get the oldest Urkel Tree root state a pruning node can recover from.
+    const oldestTreeIntervalStart = oldestBlock + toNextInterval + 1;
+    const entry = await this.db.getEntryByHeight(oldestTreeIntervalStart);
+
+    try {
+      // TODO: For RPC calls, If compaction fails while compacting
+      // and we never hit syncTree, we need to shut down the node
+      // so on restart chain can recover.
+      // Error can also happen in syncTree, but that means the DB
+      // is done for. (because restart would just retry syncTree.)
+      // It's fine on open, open throwing would just stop the node.
+
+      // Rewind Urkel Tree and delete all historical state.
+      this.emit('tree compact start', entry.treeRoot, entry);
+      await this.db.compactTree(entry);
+      await this.syncTree();
+      this.emit('tree compact end', entry.treeRoot, entry);
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Reconstruct the Urkel Tree.
+   * @returns {Promise}
+   */
+
+  async reconstructTree() {
+    if (this.options.spv)
+      return;
+
+    if (this.options.prune)
+      throw new Error('Cannot reconstruct tree in pruned mode.');
+
+    const unlock = await this.locker.lock();
+
+    const treeState = await this.db.getTreeState();
+
+    if (treeState.compactionHeight === 0)
+      throw new Error('Nothing to reconstruct.');
+
+    // Compact all the way to the first block and
+    // let the syncTree do its job.
+    const entry = await this.db.getEntryByHeight(1);
+
+    try {
+      this.emit('tree reconstruct start');
+      await this.db.compactTree(entry);
+      await this.syncTree();
+      this.emit('tree reconstruct end');
+    } finally {
+      unlock();
+    }
+  }
+
   /**
    * Scan the blockchain for transactions containing specified address hashes.
    * @param {Hash} start - Block hash to start at.
@@ -3611,6 +3761,8 @@ class ChainOptions {
     this.maxOrphans = 20;
     this.checkpoints = true;
     this.chainMigrate = -1;
+    this.compactTreeOnInit = false;
+    this.compactTreeInitInterval = 10000;
 
     if (options)
       this.fromOptions(options);
@@ -3723,6 +3875,19 @@ class ChainOptions {
       this.chainMigrate = options.chainMigrate;
     }
 
+    if (options.compactTreeOnInit != null) {
+      assert(typeof options.compactTreeOnInit === 'boolean');
+      this.compactTreeOnInit = options.compactTreeOnInit;
+    }
+
+    if (options.compactTreeInitInterval != null) {
+      const {keepBlocks} = this.network.block;
+      assert(typeof options.compactTreeInitInterval === 'number');
+      assert(options.compactTreeInitInterval >= keepBlocks,
+        `compaction interval must not be smaller than ${keepBlocks}.`);
+      this.compactTreeInitInterval = options.compactTreeInitInterval;
+    }
+
     if (this.spv || this.memory)
       this.treePrefix = null;