support ghosts

lib/index.js

@@ -22,6 +22,11 @@ const PREFIX = 'set_v1__'
  *   del: Array<string>,
  *   supersedes: Array<MsgID>,
  * }} SetMsgData
+ * @typedef {{
+ *   set?: {
+ *     ghostSpan?: number
+ *   }
+ * }} Config
  */
 
 /**
@@ -75,11 +80,14 @@ function assert(check, message) {
 
 /**
  * @param {{ db: PPPPPDB | null, close: ClosableHook }} peer
- * @param {any} config
+ * @param {Config} config
  */
 function initSet(peer, config) {
   assertDBPlugin(peer)
 
+  const ghostSpan = config.set?.ghostSpan ?? 32
+  if (ghostSpan < 1) throw new Error('config.set.ghostSpan must be >= 0')
+
   //#region state
   let accountID = /** @type {string | null} */ (null)
   let loadPromise = /** @type {Promise<void> | null} */ (null)
@@ -326,20 +334,17 @@ function initSet(peer, config) {
   }
 
   /**
-   * @param {string} id
    * @param {string} subdomain
    * @param {string} value
    * @param {CB<boolean>} cb
    */
-  function add(id, subdomain, value, cb) {
+  function add(subdomain, value, cb) {
     assertDBPlugin(peer)
     assert(!!accountID, 'Cannot add to Set before loading')
-    // prettier-ignore
-    if (id !== accountID) return cb(new Error(`Cannot add to another user's Set (${id}/${subdomain})`))
 
     loaded(() => {
       assert(!!accountID, 'Cannot add to Set before loading')
-      const currentSet = readSet(id, subdomain)
+      const currentSet = readSet(accountID, subdomain)
       if (currentSet.has(value)) return cb(null, false)
       const domain = fromSubdomain(subdomain)
 
@@ -361,7 +366,7 @@ function initSet(peer, config) {
       const data = { add: [value], del: [], supersedes }
       peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => {
         // prettier-ignore
-        if (err) return cb(new Error(`Failed to create msg when adding to Set (${id}/${subdomain})`, { cause: err }))
+        if (err) return cb(new Error(`Failed to create msg when adding to Set "${subdomain}"`, { cause: err }))
         for (const [msgID, item] of toDeleteFromItemRoots) {
           itemRoots.del(subdomain, item, msgID)
         }
@@ -372,20 +377,17 @@ function initSet(peer, config) {
   }
 
   /**
-   * @param {string} id
    * @param {string} subdomain
    * @param {string} value
    * @param {CB<boolean>} cb
    */
-  function del(id, subdomain, value, cb) {
+  function del(subdomain, value, cb) {
     assertDBPlugin(peer)
     assert(!!accountID, 'Cannot add to Set before loading')
-    // prettier-ignore
-    if (id !== accountID) return cb(new Error(`Cannot delete from another user's Set (${id}/${subdomain})`))
 
     loaded(() => {
       assert(!!accountID, 'Cannot add to Set before loading')
-      const currentSet = readSet(id, subdomain)
+      const currentSet = readSet(accountID, subdomain)
       if (!currentSet.has(value)) return cb(null, false)
       const domain = fromSubdomain(subdomain)
 
@@ -401,7 +403,7 @@ function initSet(peer, config) {
       const data = { add: [], del: [value], supersedes }
       peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => {
         // prettier-ignore
-        if (err) return cb(new Error(`Failed to create msg when deleting from Set (${id}/${subdomain})`, { cause: err }))
+        if (err) return cb(new Error(`Failed to create msg when deleting from Set "${subdomain}"`, { cause: err }))
         // @ts-ignore
         cb(null, true)
       })
@@ -428,25 +430,105 @@ function initSet(peer, config) {
   }
 
   /**
-   * @param {string} id
+   * @public
+   * @param {string} tangleID
+   * @returns {number}
+   */
+  function minGhostDepth(tangleID) {
+    return Math.max(0, minRequiredDepth(tangleID) - ghostSpan)
+  }
+
+  /**
+   * @public
+   * @param {string} tangleID
+   * @returns {number}
+   */
+  function minRequiredDepth(tangleID) {
+    assertDBPlugin(peer)
+    const tangle = peer.db.getTangle(tangleID)
+
+    // prettier-ignore
+    if (!tangle || tangle.size === 0) throw new Error(`isGhostable() tangleID "${tangleID}" is empty`)
+    // prettier-ignore
+    if (!isValidSetMoot(tangle.root)) throw new Error(`minRequiredDepth() "${tangleID}" is not a Set moot`)
+
+    // Discover item roots
+    const itemRoots = new Set()
+    const msgIDs = tangle.topoSort()
+    for (const msgID of msgIDs) {
+      const msg = peer.db.get(msgID)
+      if (!msg?.data) continue
+      for (const supersededMsgID of msg.data.supersedes) {
+        itemRoots.delete(supersededMsgID)
+      }
+      itemRoots.add(msgID)
+    }
+
+    // Get minimum depth of all item roots
+    let minDepth = Infinity
+    for (const msgID of itemRoots) {
+      const depth = tangle.getDepth(msgID)
+      if (depth < minDepth) minDepth = depth
+    }
+
+    return minDepth
+  }
+
+  /**
+   * @public
+   * @param {string} subdomain
+   * @returns {string}
+   */
+  function getFeedID(subdomain) {
+    assert(!!accountID, 'Cannot getFeedID() before loading')
+    assertDBPlugin(peer)
+    const domain = fromSubdomain(subdomain)
+    return MsgV3.getMootID(accountID, domain)
+  }
+
+  /**
+   * @public
+   * @param {MsgID} ghostableMsgID
+   * @param {MsgID} tangleID
+   */
+  function isGhostable(ghostableMsgID, tangleID) {
+    if (ghostableMsgID === tangleID) return false
+
+    assertDBPlugin(peer)
+    const msg = peer.db.get(ghostableMsgID)
+
+    // prettier-ignore
+    if (!msg) throw new Error(`isGhostable() msgID "${ghostableMsgID}" does not exist in the database`)
+
+    const minItemRootDepth = minRequiredDepth(tangleID)
+    const minGhostDepth = minItemRootDepth - ghostSpan
+    const msgDepth = msg.metadata.tangles[tangleID].depth
+    if (minGhostDepth <= msgDepth && msgDepth < minItemRootDepth) return true
+    return false
+  }
+
+  /**
+   * @returns {number}
+   */
+  function getGhostSpan() {
+    return ghostSpan
+  }
+
+  /**
    * @param {any} subdomain
    */
-  function getItemRoots(id, subdomain) {
+  function _getItemRoots(subdomain) {
-    // prettier-ignore
+    if (!accountID) throw new Error(`Cannot getItemRoots before loading`)
-    if (id !== accountID) throw new Error(`Cannot getItemRoots of another user's Set. (${id}/${subdomain})`)
     return itemRoots.getAll(subdomain)
   }
 
   /**
-   * @param {string} id
    * @param {string} subdomain
    * @param {CB<boolean>} cb
    */
-  function squeeze(id, subdomain, cb) {
+  function squeeze(subdomain, cb) {
     assertDBPlugin(peer)
     assert(!!accountID, 'Cannot squeeze Set before loading')
-    // prettier-ignore
-    if (id !== accountID) return cb(new Error(`Cannot squeeze another user's Set (${id}/${subdomain})`))
 
     const potential = _squeezePotential(subdomain)
     if (potential < 1) return cb(null, false)
@@ -454,7 +536,7 @@ function initSet(peer, config) {
     loaded(() => {
       assert(!!accountID, 'Cannot squeeze Set before loading')
       const domain = fromSubdomain(subdomain)
-      const currentSet = readSet(id, subdomain)
+      const currentSet = readSet(accountID, subdomain)
 
       const supersedes = []
       const currentItemRoots = itemRoots.getAll(subdomain)
@@ -465,7 +547,7 @@ function initSet(peer, config) {
       const data = { add: [...currentSet], del: [], supersedes }
       peer.db.feed.publish({ account: accountID, domain, data }, (err, rec) => {
         // prettier-ignore
-        if (err) return cb(new Error(`Failed to create msg when squeezing Set (${id}/${subdomain})`, { cause: err }))
+        if (err) return cb(new Error(`Failed to create msg when squeezing Set "${subdomain}"`, { cause: err }))
         // @ts-ignore
         cb(null, true)
       })
@@ -479,9 +561,14 @@ function initSet(peer, config) {
     del,
     has,
     values,
-    getItemRoots,
+    getFeedID,
+    isGhostable,
+    getGhostSpan,
+    minGhostDepth,
+    minRequiredDepth,
     squeeze,
 
+    _getItemRoots,
     _squeezePotential,
   }
 }

test/index.test.js

@@ -3,6 +3,7 @@ const assert = require('node:assert')
 const path = require('node:path')
 const os = require('node:os')
 const rimraf = require('rimraf')
+const MsgV3 = require('ppppp-db/msg-v3')
 const p = require('node:util').promisify
 const { createPeer } = require('./util')
 const Keypair = require('ppppp-keypair')
@@ -15,7 +16,11 @@ const aliceKeypair = Keypair.generate('ed25519', 'alice')
 let peer
 let aliceID
 test('setup', async (t) => {
-  peer = createPeer({ keypair: aliceKeypair, path: DIR })
+  peer = createPeer({
+    keypair: aliceKeypair,
+    path: DIR,
+    set: { ghostSpan: 4 },
+  })
 
   await peer.db.loaded()
 
@@ -24,6 +29,8 @@ test('setup', async (t) => {
     _nonce: 'alice',
   })
   await p(peer.set.load)(aliceID)
+
+  assert.equal(peer.set.getGhostSpan(), 4, 'getGhostSpan')
 })
 
 function lastMsgID() {
@@ -37,65 +44,93 @@ function lastMsgID() {
 let add1, add2, del1, add3, del2
 test('Set add(), del(), has()', async (t) => {
   // Add 1st
-  assert.equal(peer.set.has(aliceID, 'follows', '1st'), false, 'doesnt have 1st')
+  assert.equal(
-  assert(await p(peer.set.add)(aliceID, 'follows', '1st'), 'add 1st')
+    peer.set.has(aliceID, 'follows', '1st'),
+    false,
+    'doesnt have 1st'
+  )
+  assert(await p(peer.set.add)('follows', '1st'), 'add 1st')
   assert.equal(peer.set.has(aliceID, 'follows', '1st'), true, 'has 1st')
   add1 = lastMsgID()
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '1st': [add1] },
     'itemRoots'
   )
 
   // Add 2nd
-  assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd')
+  assert.equal(
-  assert(await p(peer.set.add)(aliceID, 'follows', '2nd'), 'add 2nd')
+    peer.set.has(aliceID, 'follows', '2nd'),
+    false,
+    'doesnt have 2nd'
+  )
+  assert(await p(peer.set.add)('follows', '2nd'), 'add 2nd')
   assert.equal(peer.set.has(aliceID, 'follows', '2nd'), true, 'has 2nd')
   add2 = lastMsgID()
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '1st': [add1], '2nd': [add2] },
     'itemRoots'
   )
 
   // Del 1st
   assert.equal(peer.set.has(aliceID, 'follows', '1st'), true, 'has 1st')
-  assert(await p(peer.set.del)(aliceID, 'follows', '1st'), 'del 1st')
+  assert(await p(peer.set.del)('follows', '1st'), 'del 1st')
-  assert.equal(peer.set.has(aliceID, 'follows', '1st'), false, 'doesnt have 1st')
+  assert.equal(
+    peer.set.has(aliceID, 'follows', '1st'),
+    false,
+    'doesnt have 1st'
+  )
   del1 = lastMsgID()
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '1st': [del1], '2nd': [add2] },
     'itemRoots'
   )
 
   // Add 3rd
-  assert.equal(peer.set.has(aliceID, 'follows', '3rd'), false, 'doesnt have 3rd')
+  assert.equal(
-  assert(await p(peer.set.add)(aliceID, 'follows', '3rd'), 'add 3rd')
+    peer.set.has(aliceID, 'follows', '3rd'),
+    false,
+    'doesnt have 3rd'
+  )
+  assert(await p(peer.set.add)('follows', '3rd'), 'add 3rd')
   assert.equal(peer.set.has(aliceID, 'follows', '3rd'), true, 'has 3rd')
   add3 = lastMsgID()
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '3rd': [add3], '2nd': [add2] },
     'itemRoots'
   )
 
   // Del 2nd
   assert.equal(peer.set.has(aliceID, 'follows', '2nd'), true, 'has 2nd')
-  assert(await p(peer.set.del)(aliceID, 'follows', '2nd'), 'del 2nd') // msg seq 4
+  assert(await p(peer.set.del)('follows', '2nd'), 'del 2nd') // msg seq 4
-  assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd')
+  assert.equal(
+    peer.set.has(aliceID, 'follows', '2nd'),
+    false,
+    'doesnt have 2nd'
+  )
   del2 = lastMsgID()
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '3rd': [add3], '2nd': [del2] },
     'itemRoots'
   )
 
   // Del 2nd (idempotent)
-  assert.equal(await p(peer.set.del)(aliceID, 'follows', '2nd'), false, 'del 2nd idempotent')
+  assert.equal(
-  assert.equal(peer.set.has(aliceID, 'follows', '2nd'), false, 'doesnt have 2nd')
+    await p(peer.set.del)('follows', '2nd'),
+    false,
+    'del 2nd idempotent'
+  )
+  assert.equal(
+    peer.set.has(aliceID, 'follows', '2nd'),
+    false,
+    'doesnt have 2nd'
+  )
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '3rd': [add3], '2nd': [del2] },
     'itemRoots'
   )
@@ -103,9 +138,9 @@ test('Set add(), del(), has()', async (t) => {
 
 let add4, add5
 test('Set values()', async (t) => {
-  assert(await p(peer.set.add)(aliceID, 'follows', '4th'), 'add 4th')
+  assert(await p(peer.set.add)('follows', '4th'), 'add 4th')
   add4 = lastMsgID()
-  assert(await p(peer.set.add)(aliceID, 'follows', '5th'), 'add 5th')
+  assert(await p(peer.set.add)('follows', '5th'), 'add 5th')
   add5 = lastMsgID()
 
   const expected = new Set(['3rd', '4th', '5th'])
@@ -118,29 +153,62 @@ test('Set values()', async (t) => {
 
 test('predsl Set squeeze', async (t) => {
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '3rd': [add3], '4th': [add4], '5th': [add5] },
     'itemRoots before squeeze'
   )
 
   assert.equal(peer.set._squeezePotential('follows'), 3, 'squeezePotential=3')
 
-  assert.equal(await p(peer.set.squeeze)(aliceID, 'follows'), true, 'squeezed')
+  assert.equal(await p(peer.set.squeeze)('follows'), true, 'squeezed')
   const squeezed = lastMsgID()
 
   assert.equal(peer.set._squeezePotential('follows'), 0, 'squeezePotential=0')
 
   assert.deepEqual(
-    peer.set.getItemRoots(aliceID, 'follows'),
+    peer.set._getItemRoots('follows'),
     { '3rd': [squeezed], '4th': [squeezed], '5th': [squeezed] },
     'itemRoots after squeeze'
   )
 
-  assert.equal(await p(peer.set.squeeze)(aliceID, 'follows'), false, 'squeeze again idempotent')
+  assert.equal(
+    await p(peer.set.squeeze)('follows'),
+    false,
+    'squeeze again idempotent'
+  )
   const squeezed2 = lastMsgID()
   assert.equal(squeezed, squeezed2, 'squeezed msgID is same')
 })
 
+test('Set isGhostable', (t) => {
+  const moot = MsgV3.createMoot(aliceID, 'set_v1__follows', aliceKeypair)
+  const mootID = MsgV3.getMsgID(moot)
+
+  assert.equal(mootID, peer.set.getFeedID('follows'), 'getFeedID')
+
+  const tangle = peer.db.getTangle(mootID)
+  const msgIDs = tangle.topoSort()
+
+  const itemRoots = peer.set._getItemRoots('follows')
+  assert.deepEqual(itemRoots, {
+    '3rd': [msgIDs[8]],
+    '4th': [msgIDs[8]],
+    '5th': [msgIDs[8]],
+  })
+
+  // Remember from the setup, that ghostSpan=4
+  assert.equal(msgIDs.length, 9)
+  assert.equal(peer.set.isGhostable(msgIDs[0], mootID), false) // moot
+  assert.equal(peer.set.isGhostable(msgIDs[1], mootID), false)
+  assert.equal(peer.set.isGhostable(msgIDs[2], mootID), false)
+  assert.equal(peer.set.isGhostable(msgIDs[3], mootID), false)
+  assert.equal(peer.set.isGhostable(msgIDs[4], mootID), true) // in ghostSpan
+  assert.equal(peer.set.isGhostable(msgIDs[5], mootID), true) // in ghostSpan
+  assert.equal(peer.set.isGhostable(msgIDs[6], mootID), true) // in ghostSpan
+  assert.equal(peer.set.isGhostable(msgIDs[7], mootID), true) // in ghostSpan
+  assert.equal(peer.set.isGhostable(msgIDs[8], mootID), false) // item root
+})
+
 test('teardown', async (t) => {
   await p(peer.close)(true)
 })