diff --git a/bin/neutrino b/bin/neutrino index df3b5fdb7..d42ec71c0 100755 --- a/bin/neutrino +++ b/bin/neutrino @@ -5,8 +5,6 @@ console.log('Starting bcoin'); process.title = 'bcoin'; const Neutrino = require('../lib/node/neutrino'); -const Outpoint = require('../lib/primitives/outpoint'); -const assert = require('assert'); // Doubt in db const node = new Neutrino({ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 1443e3170..532ccb050 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -35,6 +35,7 @@ const layout = { R: bdb.key('R'), D: bdb.key('D'), N: bdb.key('N'), + F: bdb.key('H', ['hash256']), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/indexer/filterindexer.js b/lib/indexer/filterindexer.js index 97265253b..ae88af139 100644 --- a/lib/indexer/filterindexer.js +++ b/lib/indexer/filterindexer.js @@ -85,6 +85,49 @@ class FilterIndexer extends Indexer { this.put(layout.f.encode(hash), gcsFilter.hash()); } + /** + * save filter header + * @param {Hash} blockHash + * @param {Hash} filterHeader + * @param {Hash} filterHash + * @returns {Promise} + */ + + async saveFilterHeader(blockHash, filterHeader, filterHash) { + assert(blockHash); + assert(filterHeader); + assert(filterHash); + + const filter = new Filter(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), filterHash); + } + + /** + * Save filter + * @param {Hash} blockHash + * @param {BasicFilter} basicFilter + * @param {Hash} filterHeader + * @returns {Promise} + */ + + async saveFilter(blockHash, basicFilter, filterHeader) { + assert(blockHash); + assert(basicFilter); + assert(filterHeader); + + const filter = new Filter(); + filter.filter = basicFilter.toRaw(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), basicFilter.hash()); + } + /** * Prune compact filters. * @private diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index 97d85f76b..c095584ca 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -292,6 +292,11 @@ class Indexer extends EventEmitter { */ async _syncBlock(meta, block, view) { + if (this.chain.options.neutrino) { + if (!this.batch) + this.start(); + return true; + } // In the case that the next block is being // connected or the current block disconnected // use the block and view being passed directly, diff --git a/lib/net/pool.js b/lib/net/pool.js index 8878319c7..5c75a0966 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -769,7 +769,7 @@ class Pool extends EventEmitter { this.filterSyncing = true; const cFilterHeight = await this.chain.getCFilterHeight(); const startHeight = cFilterHeight - ? cFilterHeight : 0; + ? cFilterHeight : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); @@ -931,6 +931,8 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); if (this.options.neutrino) { peer.sendGetHeaders(locator); + if (!this.syncing) + this.startFilterHeadersSync(); return true; } if (this.checkpoints) { @@ -1759,7 +1761,11 @@ class Pool extends EventEmitter { return; if (this.neutrino) { - this.startSync(); + const locator = await this.chain.getLocator(); + this.sendLocator(locator, peer); + if (!this.syncing) + this.startFilterHeadersSync(); + return; } // Request headers instead. @@ -2172,17 +2178,22 @@ class Pool extends EventEmitter { } const filterType = packet.filterType; - assert(filterType === this.getcfheadersFilterType); + + if (filterType !== this.getcfheadersFilterType) { + peer.ban(); + peer.destroy(); + return; + } + const stopHash = packet.stopHash; assert(stopHash.equals(this.getcfheadersStopHash)); let previousFilterHeader = packet.previousFilterHeader; const filterHashes = packet.filterHashes; - assert(filterHashes.length <= 2000); let blockHeight = await this.chain.getHeight(stopHash) - - filterHashes.length; + - filterHashes.length + 1; const stopHeight = await this.chain.getHeight(stopHash); for (const filterHash of filterHashes) { - assert(blockHeight < stopHeight); + assert(blockHeight <= stopHeight); const basicFilter = new BasicFilter(); basicFilter._hash = filterHash; const filterHeader = basicFilter.header(previousFilterHeader); @@ -2190,12 +2201,12 @@ class Pool extends EventEmitter { const cfHeaderEntry = new CFHeaderEntry( filterHash, lastFilterHeader.height + 1); this.cfHeaderChain.push(cfHeaderEntry); - // todo: verify the filterHeader - // todo: save the filterHeader + const blockHash = await this.chain.getHash(blockHeight); + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + await indexer.saveFilterHeader(blockHash, filterHeader, filterHash); previousFilterHeader = filterHeader; - // todo: add a function for this in chain.js - blockHeight++; await this.chain.saveCFHeaderHeight(blockHeight); + blockHeight++; } if (this.headerChain.tail.height <= stopHeight) this.emit('cfheaders'); @@ -2220,20 +2231,31 @@ class Pool extends EventEmitter { const filterType = packet.filterType; const filter = packet.filterBytes; - // todo: verify the filter - assert(filterType === this.getcfheadersFilterType); + if (filterType !== this.getcfheadersFilterType) { + peer.ban(); + peer.destroy(); + return; + } + const blockHeight = await this.chain.getHeight(blockHash); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); - await this.chain.saveCFilterHeight(blockHeight); - const cFilterHeight = await this.chain.getCFilterHeight(); - // todo: save the filter - const basicFilter = new BasicFilter(); - const gcsFilter = basicFilter.fromNBytes(filter); - this.emit('cfilter', blockHash, gcsFilter); + const cFilterHeight = await this.chain.getCFilterHeight(); + + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + + const filterHeader = await indexer.getFilterHeader(blockHash); + + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); + + await indexer.saveFilter(blockHash, gcsFilter, filterHeader); + + this.emit('cfilter', blockHash, gcsFilter); + await this.chain.saveCFilterHeight(blockHeight); const startHeight = stopHeight + 1; let nextStopHeight; if (cFilterHeight === stopHeight @@ -2436,7 +2458,7 @@ class Pool extends EventEmitter { this.logger.warning( 'Peer sent a bad header chain (%s).', peer.hostname()); - peer.destroy(); + peer.increaseBan(10); return; } @@ -2459,7 +2481,7 @@ class Pool extends EventEmitter { this.headerChain.push(node); if (this.options.neutrino) - await this._addBlock(peer, header, chainCommon.flags.VERIFY_NONE); + await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); } this.logger.debug( @@ -2472,7 +2494,7 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); // Request the blocks we just added. - if (checkpoint) { + if (checkpoint && !this.options.neutrino) { this.headerChain.shift(); this.resolveHeaders(peer); return; diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index d3a2c7671..4c7ccf663 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -91,6 +91,7 @@ class Neutrino extends Node { chain: this.chain, prefix: this.config.prefix, checkpoints: true, + filterIndexers: this.filterIndexers, proxy: this.config.str('proxy'), onion: this.config.bool('onion'), upnp: this.config.bool('upnp'), @@ -168,12 +169,13 @@ class Neutrino extends Node { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); - // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later await this.pool.startFilterHeadersSync(); }); this.pool.on('cfheaders', async () => { - this.logger.info('CF Headers Synced'); + if (this.chain.height === 0) + return; + this.logger.info('Filter Headers are fully synced'); await this.pool.startFilterSync(); }); @@ -200,6 +202,10 @@ class Neutrino extends Node { await this.http.open(); await this.handleOpen(); + for (const filterindex of this.filterIndexers.values()) { + await filterindex.open(); + } + this.logger.info('Node is loaded.'); } @@ -220,6 +226,10 @@ class Neutrino extends Node { await this.pool.close(); await this.chain.close(); await this.handleClose(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.close(); + } } /** diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 53b1f11da..a4134caf9 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -26,6 +26,7 @@ const Outpoint = require('../primitives/outpoint'); const layouts = require('./layout'); const records = require('./records'); const NullClient = require('./nullclient'); +const Script = require('../script/script'); const layout = layouts.wdb; const tlayout = layouts.txdb; @@ -65,6 +66,7 @@ class WalletDB extends EventEmitter { this.state = new ChainState(); this.confirming = false; this.height = 0; + this.filterHeight = 0; this.wallets = new Map(); this.depth = 0; this.rescanning = false; @@ -577,7 +579,9 @@ class WalletDB extends EventEmitter { } async checkFilter (blockHash, filter) { - const gcsKey = blockHash.slice(0, 16); + // script pub keys + this.filterHeight = this.filterHeight + 1; + const gcsKey = blockHash.reverse().slice(0, 16); const piter = this.db.iterator({ gte: layout.p.min(), @@ -586,26 +590,14 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); - const match = filter.match(gcsKey, data); - if (match) { - await this.client.getBlockFromNode(blockHash, filter); - return; - } - }); - - const oiter = this.db.iterator({ - gte: layout.o.min(), - lte: layout.o.max() - }); - - await oiter.each(async (key) => { - const [hash, index] = layout.o.decode(key); - const outpoint = new Outpoint(hash, index); - const data = outpoint.toRaw(); - const match = filter.match(gcsKey, data); - if (match) { - await this.client.getBlockFromNode(blockHash, filter); - return; + // address fromHash toScript + if (data.length === 20) { + const script = Script.fromPubkeyhash(data); + const match = filter.match(gcsKey, script); + if (match) { + await this.client.getBlockFromNode(blockHash, filter); + return; + } } }); } diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index 2385ada5e..f0be7547c 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -4,8 +4,10 @@ const FullNode = require('../lib/node/fullnode'); const Neutrino = require('../lib/node/neutrino'); const MTX = require('../lib/primitives/mtx'); const assert = require('bsert'); -const { consensus } = require('../lib/protocol'); const { forValue } = require('./util/common'); +const BasicFilter = require('../lib/golomb/basicFilter'); +const Script = require('../lib/script/script'); +const Address = require('../lib/primitives/address'); const node1 = new FullNode({ network: 'regtest', @@ -36,9 +38,11 @@ const wdb2 = node2.require('walletdb').wdb; let wallet1 = null; let wallet2 = null; -let cb = null; +const fwAddresses = []; +const nwAddresses = []; -async function mineBlock(tx) { +async function mineBlock(tx, address) { + console.log('address', address); const job = await miner.createJob(); if (!tx) @@ -46,9 +50,7 @@ async function mineBlock(tx) { const spend = new MTX(); spend.addTX(tx, 0); - - spend.addOutput(await wallet2.receiveAddress(), 25 * 1e8); - spend.addOutput(await wallet2.changeAddress(), 5 * 1e8); + spend.addOutput(address, 50000); spend.setLocktime(chain.height); await wallet1.sign(spend); @@ -62,7 +64,6 @@ async function mineBlock(tx) { describe('wallet-neutrino', function() { it('should open chain and miner', async () => { miner.mempool = null; - consensus.COINBASE_MATURITY = 0; await node1.open(); await node2.open(); }); @@ -70,18 +71,42 @@ describe('wallet-neutrino', function() { it('should open walletdb', async () => { wallet1 = await wdb1.create(); wallet2 = await wdb2.create(); - miner.addresses.length = 0; - miner.addAddress(await wallet1.receiveAddress()); + }); + + it('should create accounts', async () => { + await wallet1.createAccount('fw'); + await wallet2.createAccount('nw'); + }); + + it('should generate addresses', async () => { + miner.addresses.length = 0; + for (let i = 0; i < 10; i++) { + const key = await wallet1.createReceive(0); + const address = key.getAddress().toString(node1.network.type); + // console.log(address); + fwAddresses.push(address); + miner.addAddress(address); + } + for (let i = 0; i < 10; i++) { + const key = await wallet2.createReceive(0); + const address = key.getAddress().toString(node2.network.type); + nwAddresses.push(address); + } }); it('should mine 10 blocks', async () => { - let n = 10; - while (n) { - const block = await mineBlock(cb); - cb = block.txs[0]; - await node1.chain.add(block); - n--; + for (const address of fwAddresses) { + for (let i = 0; i < 2; i++) { + const block = await mineBlock(null, address); + await chain.add(block); } + } + for (const address of nwAddresses) { + for (let i = 0; i < 2; i++) { + const block = await mineBlock(null, address); + await chain.add(block); + } + } }); it('should connect nodes', async () => { @@ -110,4 +135,24 @@ describe('wallet-neutrino', function() { const filterHeight = await node2.chain.getCFilterHeight(); assert.equal(filterHeight, node2.chain.height); }); + + it('should send filters to wallet', async () => { + assert.equal(wdb2.filterHeight, node2.chain.height); + }); + + it('should match the filters', async () => { + const filterIndexer = node2.filterIndexers.get('BASIC'); + for (let i = 0; i < fwAddresses.length; i++) { + const hash = await node2.chain.getHash(i); + const filter = await filterIndexer.getFilter(hash); + const basicFilter = new BasicFilter(); + const gcs = basicFilter.fromNBytes(filter.filter); + const key = hash.slice(0, 16); + const address = Address.fromString(fwAddresses[i], node2.network.type); + const script = Script.fromAddress(address); + // console.log(address.hash); + console.log(script.toRaw()); + // assert(gcs.match(key, script.)); + } + }); });