feat: save filters and added wallet test

Author: manavdesai27

bin/neutrino

@@ -5,8 +5,6 @@
 console.log('Starting bcoin');
 process.title = 'bcoin';
 const Neutrino = require('../lib/node/neutrino');
-const Outpoint = require('../lib/primitives/outpoint');
-const assert = require('assert');
 
 // Doubt in db
 const node = new Neutrino({

lib/blockchain/layout.js

@@ -35,6 +35,7 @@ const layout = {
   R: bdb.key('R'),
   D: bdb.key('D'),
   N: bdb.key('N'),
+  F: bdb.key('H', ['hash256']),
   e: bdb.key('e', ['hash256']),
   h: bdb.key('h', ['hash256']),
   H: bdb.key('H', ['uint32']),

lib/indexer/filterindexer.js

@@ -85,6 +85,49 @@ class FilterIndexer extends Indexer {
     this.put(layout.f.encode(hash), gcsFilter.hash());
   }
 
+  /**
+   * save filter header
+   * @param {Hash} blockHash
+   * @param {Hash} filterHeader
+   * @param {Hash} filterHash
+   * @returns {Promise}
+   */
+
+  async saveFilterHeader(blockHash, filterHeader, filterHash) {
+    assert(blockHash);
+    assert(filterHeader);
+    assert(filterHash);
+
+    const filter = new Filter();
+    filter.header = filterHeader;
+
+    await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType);
+    // console.log(layout.f.encode(blockHash));
+    this.put(layout.f.encode(blockHash), filterHash);
+  }
+
+   /**
+   * Save filter
+   * @param {Hash} blockHash
+   * @param {BasicFilter} basicFilter
+   * @param {Hash} filterHeader
+   * @returns {Promise}
+   */
+
+   async saveFilter(blockHash, basicFilter, filterHeader) {
+    assert(blockHash);
+    assert(basicFilter);
+    assert(filterHeader);
+
+    const filter = new Filter();
+    filter.filter = basicFilter.toRaw();
+    filter.header = filterHeader;
+
+    await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType);
+    // console.log(layout.f.encode(blockHash));
+    this.put(layout.f.encode(blockHash), basicFilter.hash());
+   }
+
   /**
    * Prune compact filters.
    * @private

lib/indexer/indexer.js

@@ -292,6 +292,11 @@ class Indexer extends EventEmitter {
    */
 
   async _syncBlock(meta, block, view) {
+    if (this.chain.options.neutrino) {
+      if (!this.batch)
+        this.start();
+      return true;
+    }
     // In the case that the next block is being
     // connected or the current block disconnected
     // use the block and view being passed directly,

lib/net/pool.js

@@ -769,7 +769,7 @@ class Pool extends EventEmitter {
     this.filterSyncing = true;
     const cFilterHeight = await this.chain.getCFilterHeight();
     const startHeight = cFilterHeight
-    ? cFilterHeight : 0;
+    ? cFilterHeight : 1;
     const chainHeight = await this.chain.height;
     const stopHeight = chainHeight > 1000 ? 1000 : chainHeight;
     const stopHash = await this.chain.getHash(stopHeight);
@@ -931,6 +931,8 @@ class Pool extends EventEmitter {
     peer.blockTime = Date.now();
     if (this.options.neutrino) {
       peer.sendGetHeaders(locator);
+      if (!this.syncing)
+        this.startFilterHeadersSync();
       return true;
     }
     if (this.checkpoints) {
@@ -1759,7 +1761,11 @@ class Pool extends EventEmitter {
       return;
 
     if (this.neutrino) {
-      this.startSync();
+      const locator = await this.chain.getLocator();
+      this.sendLocator(locator, peer);
+      if (!this.syncing)
+        this.startFilterHeadersSync();
+      return;
     }
 
     // Request headers instead.
@@ -2172,30 +2178,35 @@ class Pool extends EventEmitter {
     }
 
     const filterType = packet.filterType;
-    assert(filterType === this.getcfheadersFilterType);
+
+    if (filterType !== this.getcfheadersFilterType) {
+      peer.ban();
+      peer.destroy();
+      return;
+    }
+
     const stopHash = packet.stopHash;
     assert(stopHash.equals(this.getcfheadersStopHash));
     let previousFilterHeader = packet.previousFilterHeader;
     const filterHashes = packet.filterHashes;
-    assert(filterHashes.length <= 2000);
     let blockHeight = await this.chain.getHeight(stopHash)
-      - filterHashes.length;
+      - filterHashes.length + 1;
     const stopHeight = await this.chain.getHeight(stopHash);
     for (const filterHash of filterHashes) {
-      assert(blockHeight < stopHeight);
+      assert(blockHeight <= stopHeight);
       const basicFilter = new BasicFilter();
       basicFilter._hash = filterHash;
       const filterHeader = basicFilter.header(previousFilterHeader);
       const lastFilterHeader = this.cfHeaderChain.tail;
       const cfHeaderEntry = new CFHeaderEntry(
         filterHash, lastFilterHeader.height + 1);
       this.cfHeaderChain.push(cfHeaderEntry);
-      // todo: verify the filterHeader
-      // todo: save the filterHeader
+      const blockHash = await this.chain.getHash(blockHeight);
+      const indexer = this.getFilterIndexer(filtersByVal[filterType]);
+      await indexer.saveFilterHeader(blockHash, filterHeader, filterHash);
       previousFilterHeader = filterHeader;
-      // todo: add a function for this in chain.js
-      blockHeight++;
       await this.chain.saveCFHeaderHeight(blockHeight);
+      blockHeight++;
     }
     if (this.headerChain.tail.height <= stopHeight)
       this.emit('cfheaders');
@@ -2220,20 +2231,31 @@ class Pool extends EventEmitter {
     const filterType = packet.filterType;
     const filter = packet.filterBytes;
 
-    // todo: verify the filter
-    assert(filterType === this.getcfheadersFilterType);
+    if (filterType !== this.getcfheadersFilterType) {
+      peer.ban();
+      peer.destroy();
+      return;
+    }
+
     const blockHeight = await this.chain.getHeight(blockHash);
     const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash);
 
     assert(blockHeight >= this.getcfiltersStartHeight
       && blockHeight <= stopHeight);
 
-    await this.chain.saveCFilterHeight(blockHeight);
-    const cFilterHeight = await this.chain.getCFilterHeight();
-    // todo: save the filter
-    const basicFilter = new BasicFilter();
-    const gcsFilter = basicFilter.fromNBytes(filter);
-    this.emit('cfilter', blockHash, gcsFilter);
+      const cFilterHeight = await this.chain.getCFilterHeight();
+
+      const indexer = this.getFilterIndexer(filtersByVal[filterType]);
+
+      const filterHeader = await indexer.getFilterHeader(blockHash);
+
+      const basicFilter = new BasicFilter();
+      const gcsFilter = basicFilter.fromNBytes(filter);
+
+      await indexer.saveFilter(blockHash, gcsFilter, filterHeader);
+
+      this.emit('cfilter', blockHash, gcsFilter);
+      await this.chain.saveCFilterHeight(blockHeight);
     const startHeight = stopHeight + 1;
     let nextStopHeight;
     if (cFilterHeight === stopHeight
@@ -2436,7 +2458,7 @@ class Pool extends EventEmitter {
         this.logger.warning(
           'Peer sent a bad header chain (%s).',
           peer.hostname());
-        peer.destroy();
+        peer.increaseBan(10);
         return;
       }
 
@@ -2459,7 +2481,7 @@ class Pool extends EventEmitter {
 
       this.headerChain.push(node);
       if (this.options.neutrino)
-        await this._addBlock(peer, header, chainCommon.flags.VERIFY_NONE);
+        await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW);
     }
 
     this.logger.debug(
@@ -2472,7 +2494,7 @@ class Pool extends EventEmitter {
     peer.blockTime = Date.now();
 
     // Request the blocks we just added.
-    if (checkpoint) {
+    if (checkpoint && !this.options.neutrino) {
       this.headerChain.shift();
       this.resolveHeaders(peer);
       return;

lib/node/neutrino.js

@@ -91,6 +91,7 @@ class Neutrino extends Node {
       chain: this.chain,
       prefix: this.config.prefix,
       checkpoints: true,
+      filterIndexers: this.filterIndexers,
       proxy: this.config.str('proxy'),
       onion: this.config.bool('onion'),
       upnp: this.config.bool('upnp'),
@@ -168,12 +169,13 @@ class Neutrino extends Node {
       if (this.chain.height === 0)
         return;
       this.logger.info('Block Headers are fully synced');
-      // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later
       await this.pool.startFilterHeadersSync();
     });
 
     this.pool.on('cfheaders', async () => {
-      this.logger.info('CF Headers Synced');
+      if (this.chain.height === 0)
+        return;
+      this.logger.info('Filter Headers are fully synced');
       await this.pool.startFilterSync();
     });
 
@@ -200,6 +202,10 @@ class Neutrino extends Node {
     await this.http.open();
     await this.handleOpen();
 
+    for (const filterindex of this.filterIndexers.values()) {
+      await filterindex.open();
+    }
+
     this.logger.info('Node is loaded.');
   }
 
@@ -220,6 +226,10 @@ class Neutrino extends Node {
     await this.pool.close();
     await this.chain.close();
     await this.handleClose();
+
+    for (const filterindex of this.filterIndexers.values()) {
+      await filterindex.close();
+    }
   }
 
   /**

lib/wallet/walletdb.js

@@ -26,6 +26,7 @@ const Outpoint = require('../primitives/outpoint');
 const layouts = require('./layout');
 const records = require('./records');
 const NullClient = require('./nullclient');
+const Script = require('../script/script');
 const layout = layouts.wdb;
 const tlayout = layouts.txdb;
 
@@ -65,6 +66,7 @@ class WalletDB extends EventEmitter {
     this.state = new ChainState();
     this.confirming = false;
     this.height = 0;
+    this.filterHeight = 0;
     this.wallets = new Map();
     this.depth = 0;
     this.rescanning = false;
@@ -577,7 +579,9 @@ class WalletDB extends EventEmitter {
   }
 
   async checkFilter (blockHash, filter) {
-    const gcsKey = blockHash.slice(0, 16);
+    // script pub keys
+    this.filterHeight = this.filterHeight + 1;
+    const gcsKey = blockHash.reverse().slice(0, 16);
 
     const piter = this.db.iterator({
       gte: layout.p.min(),
@@ -586,26 +590,14 @@ class WalletDB extends EventEmitter {
 
     await piter.each(async (key) => {
       const [data] = layout.p.decode(key);
-      const match = filter.match(gcsKey, data);
-      if (match) {
-        await this.client.getBlockFromNode(blockHash, filter);
-        return;
-      }
-    });
-
-    const oiter = this.db.iterator({
-      gte: layout.o.min(),
-      lte: layout.o.max()
-    });
-
-    await oiter.each(async (key) => {
-      const [hash, index] = layout.o.decode(key);
-      const outpoint = new Outpoint(hash, index);
-      const data = outpoint.toRaw();
-      const match = filter.match(gcsKey, data);
-      if (match) {
-        await this.client.getBlockFromNode(blockHash, filter);
-        return;
+      // address fromHash toScript
+      if (data.length === 20) {
+        const script = Script.fromPubkeyhash(data);
+        const match = filter.match(gcsKey, script);
+        if (match) {
+          await this.client.getBlockFromNode(blockHash, filter);
+          return;
+        }
       }
     });
   }