From 3e32267b74e663c09ec6cddbbab2f829fe0ed168 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 26 Oct 2023 17:12:04 +0300 Subject: [PATCH] support ghosts --- lib/index.js | 135 +++++++++++++++++++++++++++++++++++++-------- test/index.test.js | 118 ++++++++++++++++++++++++++++++--------- 2 files changed, 204 insertions(+), 49 deletions(-) diff --git a/lib/index.js b/lib/index.js index 0db8b30..39cd089 100644 --- a/lib/index.js +++ b/lib/index.js @@ -22,6 +22,11 @@ const PREFIX = 'set_v1__' * del: Array, * supersedes: Array, * }} SetMsgData + * @typedef {{ + * set?: { + * ghostSpan?: number + * } + * }} Config */ /** @@ -75,11 +80,14 @@ function assert(check, message) { /** * @param {{ db: PPPPPDB | null, close: ClosableHook }} peer - * @param {any} config + * @param {Config} config */ function initSet(peer, config) { assertDBPlugin(peer) + const ghostSpan = config.set?.ghostSpan ?? 32 + if (ghostSpan < 1) throw new Error('config.set.ghostSpan must be >= 0') + //#region state let accountID = /** @type {string | null} */ (null) let loadPromise = /** @type {Promise | null} */ (null) @@ -326,20 +334,17 @@ function initSet(peer, config) { } /** - * @param {string} id * @param {string} subdomain * @param {string} value * @param {CB} cb */ - function add(id, subdomain, value, cb) { + function add(subdomain, value, cb) { assertDBPlugin(peer) assert(!!accountID, 'Cannot add to Set before loading') - // prettier-ignore - if (id !== accountID) return cb(new Error(`Cannot add to another user's Set (${id}/${subdomain})`)) loaded(() => { assert(!!accountID, 'Cannot add to Set before loading') - const currentSet = readSet(id, subdomain) + const currentSet = readSet(accountID, subdomain) if (currentSet.has(value)) return cb(null, false) const domain = fromSubdomain(subdomain) @@ -361,7 +366,7 @@ function initSet(peer, config) { const data = { add: [value], del: [], supersedes } peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => { // prettier-ignore - if (err) return cb(new Error(`Failed to create msg when adding to Set (${id}/${subdomain})`, { cause: err })) + if (err) return cb(new Error(`Failed to create msg when adding to Set "${subdomain}"`, { cause: err })) for (const [msgID, item] of toDeleteFromItemRoots) { itemRoots.del(subdomain, item, msgID) } @@ -372,20 +377,17 @@ function initSet(peer, config) { } /** - * @param {string} id * @param {string} subdomain * @param {string} value * @param {CB} cb */ - function del(id, subdomain, value, cb) { + function del(subdomain, value, cb) { assertDBPlugin(peer) assert(!!accountID, 'Cannot add to Set before loading') - // prettier-ignore - if (id !== accountID) return cb(new Error(`Cannot delete from another user's Set (${id}/${subdomain})`)) loaded(() => { assert(!!accountID, 'Cannot add to Set before loading') - const currentSet = readSet(id, subdomain) + const currentSet = readSet(accountID, subdomain) if (!currentSet.has(value)) return cb(null, false) const domain = fromSubdomain(subdomain) @@ -401,7 +403,7 @@ function initSet(peer, config) { const data = { add: [], del: [value], supersedes } peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => { // prettier-ignore - if (err) return cb(new Error(`Failed to create msg when deleting from Set (${id}/${subdomain})`, { cause: err })) + if (err) return cb(new Error(`Failed to create msg when deleting from Set "${subdomain}"`, { cause: err })) // @ts-ignore cb(null, true) }) @@ -428,25 +430,105 @@ function initSet(peer, config) { } /** - * @param {string} id + * @public + * @param {string} tangleID + * @returns {number} + */ + function minGhostDepth(tangleID) { + return Math.max(0, minRequiredDepth(tangleID) - ghostSpan) + } + + /** + * @public + * @param {string} tangleID + * @returns {number} + */ + function minRequiredDepth(tangleID) { + assertDBPlugin(peer) + const tangle = peer.db.getTangle(tangleID) + + // prettier-ignore + if (!tangle || tangle.size === 0) throw new Error(`isGhostable() tangleID "${tangleID}" is empty`) + // prettier-ignore + if (!isValidSetMoot(tangle.root)) throw new Error(`minRequiredDepth() "${tangleID}" is not a Set moot`) + + // Discover item roots + const itemRoots = new Set() + const msgIDs = tangle.topoSort() + for (const msgID of msgIDs) { + const msg = peer.db.get(msgID) + if (!msg?.data) continue + for (const supersededMsgID of msg.data.supersedes) { + itemRoots.delete(supersededMsgID) + } + itemRoots.add(msgID) + } + + // Get minimum depth of all item roots + let minDepth = Infinity + for (const msgID of itemRoots) { + const depth = tangle.getDepth(msgID) + if (depth < minDepth) minDepth = depth + } + + return minDepth + } + + /** + * @public + * @param {string} subdomain + * @returns {string} + */ + function getFeedID(subdomain) { + assert(!!accountID, 'Cannot getFeedID() before loading') + assertDBPlugin(peer) + const domain = fromSubdomain(subdomain) + return MsgV3.getMootID(accountID, domain) + } + + /** + * @public + * @param {MsgID} ghostableMsgID + * @param {MsgID} tangleID + */ + function isGhostable(ghostableMsgID, tangleID) { + if (ghostableMsgID === tangleID) return false + + assertDBPlugin(peer) + const msg = peer.db.get(ghostableMsgID) + + // prettier-ignore + if (!msg) throw new Error(`isGhostable() msgID "${ghostableMsgID}" does not exist in the database`) + + const minItemRootDepth = minRequiredDepth(tangleID) + const minGhostDepth = minItemRootDepth - ghostSpan + const msgDepth = msg.metadata.tangles[tangleID].depth + if (minGhostDepth <= msgDepth && msgDepth < minItemRootDepth) return true + return false + } + + /** + * @returns {number} + */ + function getGhostSpan() { + return ghostSpan + } + + /** * @param {any} subdomain */ - function getItemRoots(id, subdomain) { - // prettier-ignore - if (id !== accountID) throw new Error(`Cannot getItemRoots of another user's Set. (${id}/${subdomain})`) + function _getItemRoots(subdomain) { + if (!accountID) throw new Error(`Cannot getItemRoots before loading`) return itemRoots.getAll(subdomain) } /** - * @param {string} id * @param {string} subdomain * @param {CB} cb */ - function squeeze(id, subdomain, cb) { + function squeeze(subdomain, cb) { assertDBPlugin(peer) assert(!!accountID, 'Cannot squeeze Set before loading') - // prettier-ignore - if (id !== accountID) return cb(new Error(`Cannot squeeze another user's Set (${id}/${subdomain})`)) const potential = _squeezePotential(subdomain) if (potential < 1) return cb(null, false) @@ -454,7 +536,7 @@ function initSet(peer, config) { loaded(() => { assert(!!accountID, 'Cannot squeeze Set before loading') const domain = fromSubdomain(subdomain) - const currentSet = readSet(id, subdomain) + const currentSet = readSet(accountID, subdomain) const supersedes = [] const currentItemRoots = itemRoots.getAll(subdomain) @@ -465,7 +547,7 @@ function initSet(peer, config) { const data = { add: [...currentSet], del: [], supersedes } peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => { // prettier-ignore - if (err) return cb(new Error(`Failed to create msg when squeezing Set (${id}/${subdomain})`, { cause: err })) + if (err) return cb(new Error(`Failed to create msg when squeezing Set "${subdomain}"`, { cause: err })) // @ts-ignore cb(null, true) }) @@ -479,9 +561,14 @@ function initSet(peer, config) { del, has, values, - getItemRoots, + getFeedID, + isGhostable, + getGhostSpan, + minGhostDepth, + minRequiredDepth, squeeze, + _getItemRoots, _squeezePotential, } } diff --git a/test/index.test.js b/test/index.test.js index dbac05d..6454214 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -3,6 +3,7 @@ const assert = require('node:assert') const path = require('node:path') const os = require('node:os') const rimraf = require('rimraf') +const MsgV3 = require('ppppp-db/msg-v3') const p = require('node:util').promisify const { createPeer } = require('./util') const Keypair = require('ppppp-keypair') @@ -15,7 +16,11 @@ const aliceKeypair = Keypair.generate('ed25519', 'alice') let peer let aliceID test('setup', async (t) => { - peer = createPeer({ keypair: aliceKeypair, path: DIR }) + peer = createPeer({ + keypair: aliceKeypair, + path: DIR, + set: { ghostSpan: 4 }, + }) await peer.db.loaded() @@ -24,6 +29,8 @@ test('setup', async (t) => { _nonce: 'alice', }) await p(peer.set.load)(aliceID) + + assert.equal(peer.set.getGhostSpan(), 4, 'getGhostSpan') }) function lastMsgID() { @@ -37,65 +44,93 @@ function lastMsgID() { let add1, add2, del1, add3, del2 test('Set add(), del(), has()', async (t) => { // Add 1st - assert.equal(peer.set.has(aliceID, 'follows', '1st'), false, 'doesnt have 1st') - assert(await p(peer.set.add)(aliceID, 'follows', '1st'), 'add 1st') + assert.equal( + peer.set.has(aliceID, 'follows', '1st'), + false, + 'doesnt have 1st' + ) + assert(await p(peer.set.add)('follows', '1st'), 'add 1st') assert.equal(peer.set.has(aliceID, 'follows', '1st'), true, 'has 1st') add1 = lastMsgID() assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '1st': [add1] }, 'itemRoots' ) // Add 2nd - assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd') - assert(await p(peer.set.add)(aliceID, 'follows', '2nd'), 'add 2nd') + assert.equal( + peer.set.has(aliceID, 'follows', '2nd'), + false, + 'doesnt have 2nd' + ) + assert(await p(peer.set.add)('follows', '2nd'), 'add 2nd') assert.equal(peer.set.has(aliceID, 'follows', '2nd'), true, 'has 2nd') add2 = lastMsgID() assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '1st': [add1], '2nd': [add2] }, 'itemRoots' ) // Del 1st assert.equal(peer.set.has(aliceID, 'follows', '1st'), true, 'has 1st') - assert(await p(peer.set.del)(aliceID, 'follows', '1st'), 'del 1st') - assert.equal(peer.set.has(aliceID, 'follows', '1st'), false, 'doesnt have 1st') + assert(await p(peer.set.del)('follows', '1st'), 'del 1st') + assert.equal( + peer.set.has(aliceID, 'follows', '1st'), + false, + 'doesnt have 1st' + ) del1 = lastMsgID() assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '1st': [del1], '2nd': [add2] }, 'itemRoots' ) // Add 3rd - assert.equal(peer.set.has(aliceID, 'follows', '3rd'), false, 'doesnt have 3rd') - assert(await p(peer.set.add)(aliceID, 'follows', '3rd'), 'add 3rd') + assert.equal( + peer.set.has(aliceID, 'follows', '3rd'), + false, + 'doesnt have 3rd' + ) + assert(await p(peer.set.add)('follows', '3rd'), 'add 3rd') assert.equal(peer.set.has(aliceID, 'follows', '3rd'), true, 'has 3rd') add3 = lastMsgID() assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '3rd': [add3], '2nd': [add2] }, 'itemRoots' ) // Del 2nd assert.equal(peer.set.has(aliceID, 'follows', '2nd'), true, 'has 2nd') - assert(await p(peer.set.del)(aliceID, 'follows', '2nd'), 'del 2nd') // msg seq 4 - assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd') + assert(await p(peer.set.del)('follows', '2nd'), 'del 2nd') // msg seq 4 + assert.equal( + peer.set.has(aliceID, 'follows', '2nd'), + false, + 'doesnt have 2nd' + ) del2 = lastMsgID() assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '3rd': [add3], '2nd': [del2] }, 'itemRoots' ) // Del 2nd (idempotent) - assert.equal(await p(peer.set.del)(aliceID, 'follows', '2nd'), false, 'del 2nd idempotent') - assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd') + assert.equal( + await p(peer.set.del)('follows', '2nd'), + false, + 'del 2nd idempotent' + ) + assert.equal( + peer.set.has(aliceID, 'follows', '2nd'), + false, + 'doesnt have 2nd' + ) assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '3rd': [add3], '2nd': [del2] }, 'itemRoots' ) @@ -103,9 +138,9 @@ test('Set add(), del(), has()', async (t) => { let add4, add5 test('Set values()', async (t) => { - assert(await p(peer.set.add)(aliceID, 'follows', '4th'), 'add 4th') + assert(await p(peer.set.add)('follows', '4th'), 'add 4th') add4 = lastMsgID() - assert(await p(peer.set.add)(aliceID, 'follows', '5th'), 'add 5th') + assert(await p(peer.set.add)('follows', '5th'), 'add 5th') add5 = lastMsgID() const expected = new Set(['3rd', '4th', '5th']) @@ -118,29 +153,62 @@ test('Set values()', async (t) => { test('predsl Set squeeze', async (t) => { assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '3rd': [add3], '4th': [add4], '5th': [add5] }, 'itemRoots before squeeze' ) assert.equal(peer.set._squeezePotential('follows'), 3, 'squeezePotential=3') - assert.equal(await p(peer.set.squeeze)(aliceID, 'follows'), true, 'squeezed') + assert.equal(await p(peer.set.squeeze)('follows'), true, 'squeezed') const squeezed = lastMsgID() assert.equal(peer.set._squeezePotential('follows'), 0, 'squeezePotential=0') assert.deepEqual( - peer.set.getItemRoots(aliceID, 'follows'), + peer.set._getItemRoots('follows'), { '3rd': [squeezed], '4th': [squeezed], '5th': [squeezed] }, 'itemRoots after squeeze' ) - assert.equal(await p(peer.set.squeeze)(aliceID, 'follows'), false, 'squeeze again idempotent') + assert.equal( + await p(peer.set.squeeze)('follows'), + false, + 'squeeze again idempotent' + ) const squeezed2 = lastMsgID() assert.equal(squeezed, squeezed2, 'squeezed msgID is same') }) +test('Set isGhostable', (t) => { + const moot = MsgV3.createMoot(aliceID, 'set_v1__follows', aliceKeypair) + const mootID = MsgV3.getMsgID(moot) + + assert.equal(mootID, peer.set.getFeedID('follows'), 'getFeedID') + + const tangle = peer.db.getTangle(mootID) + const msgIDs = tangle.topoSort() + + const itemRoots = peer.set._getItemRoots('follows') + assert.deepEqual(itemRoots, { + '3rd': [msgIDs[8]], + '4th': [msgIDs[8]], + '5th': [msgIDs[8]], + }) + + // Remember from the setup, that ghostSpan=4 + assert.equal(msgIDs.length, 9) + assert.equal(peer.set.isGhostable(msgIDs[0], mootID), false) // moot + assert.equal(peer.set.isGhostable(msgIDs[1], mootID), false) + assert.equal(peer.set.isGhostable(msgIDs[2], mootID), false) + assert.equal(peer.set.isGhostable(msgIDs[3], mootID), false) + assert.equal(peer.set.isGhostable(msgIDs[4], mootID), true) // in ghostSpan + assert.equal(peer.set.isGhostable(msgIDs[5], mootID), true) // in ghostSpan + assert.equal(peer.set.isGhostable(msgIDs[6], mootID), true) // in ghostSpan + assert.equal(peer.set.isGhostable(msgIDs[7], mootID), true) // in ghostSpan + assert.equal(peer.set.isGhostable(msgIDs[8], mootID), false) // item root +}) + test('teardown', async (t) => { await p(peer.close)(true) })