support goal=dict and goal=set

lib/algorithm.js

@@ -6,6 +6,7 @@ const { EMPTY_RANGE, isEmptyRange, estimateMsgCount } = require('./range')
 /**
  * @typedef {ReturnType<import('ppppp-db').init>} PPPPPDB
  * @typedef {ReturnType<import('ppppp-dict').init>} PPPPPDict
+ * @typedef {ReturnType<import('ppppp-set').init>} PPPPPSet
  * @typedef {import('ppppp-db/msg-v3').Msg} Msg
  * @typedef {import('ppppp-goals').Goal} Goal
  * @typedef {import('./range').Range} Range
@@ -31,11 +32,21 @@ function assertDictPlugin(peer) {
   }
 }
 
+/**
+ * @param {{ set: PPPPPSet | null }} peer
+ * @returns {asserts peer is { set: PPPPPSet }}
+ */
+function assertSetPlugin(peer) {
+  if (!peer.set) {
+    throw new Error('tanglesync plugin requires ppppp-set plugin')
+  }
+}
+
 class Algorithm {
-  /** @type {{ db: PPPPPDB, dict: PPPPPDict | null }} */
+  /** @type {{ db: PPPPPDB, dict: PPPPPDict | null, set: PPPPPSet | null }} */
   #peer
 
-  /** @param {{ db: PPPPPDB, dict: PPPPPDict | null }} peer */
+  /** @param {{ db: PPPPPDB, dict: PPPPPDict | null, set: PPPPPSet | null }} peer */
   constructor(peer) {
     this.#peer = peer
   }
@@ -97,7 +108,7 @@ class Algorithm {
    * @param {Range} remoteHaveRange
    * @returns {Range}
    */
-  #wantDictRange(minGhostDepth, remoteHaveRange) {
+  #wantDictOrSetRange(minGhostDepth, remoteHaveRange) {
     const [minRemoteHave, maxRemoteHave] = remoteHaveRange
     if (maxRemoteHave < minGhostDepth) return EMPTY_RANGE
     const maxWant = maxRemoteHave
@@ -121,11 +132,13 @@ class Algorithm {
 
       case 'dict':
         assertDictPlugin(this.#peer)
-        const minGhostDepth = this.#peer.dict.minGhostDepth(goal.id)
+        const minDictGhostDepth = this.#peer.dict.minGhostDepth(goal.id)
-        return this.#wantDictRange(minGhostDepth, remoteHave)
+        return this.#wantDictOrSetRange(minDictGhostDepth, remoteHave)
 
       case 'set':
-        throw new Error('Not implemented') // TODO
+        assertSetPlugin(this.#peer)
+        const minSetGhostDepth = this.#peer.set.minGhostDepth(goal.id)
+        return this.#wantDictOrSetRange(minSetGhostDepth, remoteHave)
 
       case 'newest':
         return this.#wantNewestRange(localHave, remoteHave, goal.count)

lib/index.js

@@ -7,8 +7,10 @@ const Algorithm = require('./algorithm')
 const SyncStream = require('./stream')
 
 /**
- * @typedef {ReturnType<import('ppppp-goals').init>} PPPPPGoal
  * @typedef {ReturnType<import('ppppp-db').init>} PPPPPDB
+ * @typedef {ReturnType<import('ppppp-dict').init>} PPPPPDict
+ * @typedef {ReturnType<import('ppppp-set').init>} PPPPPSet
+ * @typedef {ReturnType<import('ppppp-goals').init>} PPPPPGoals
  * @typedef {import('node:events').EventEmitter} Emitter
  * @typedef {(cb: (err: Error) => void) => import('pull-stream').Duplex<unknown, unknown>} GetDuplex
  * @typedef {{ pubkey: string }} SHSE
@@ -34,8 +36,8 @@ function assertDBExists(peer) {
 }
 
 /**
- * @param {{ goals: PPPPPGoal | null }} peer
+ * @param {{ goals: PPPPPGoals | null }} peer
- * @returns {asserts peer is { goals: PPPPPGoal }}
+ * @returns {asserts peer is { goals: PPPPPGoals }}
  */
 function assertGoalsExists(peer) {
   if (!peer.goals) throw new Error('tangleSync requires ppppp-goals plugin')
@@ -64,7 +66,9 @@ module.exports = {
   /**
    * @param {Emitter & {
    *   db: PPPPPDB | null,
-   *   goals: PPPPPGoal | null,
+   *   dict: PPPPPDict | null,
+   *   set: PPPPPSet | null,
+   *   goals: PPPPPGoals | null,
    *   shse: SHSE | null
    * }} peer
    * @param {unknown} config

package.json

@@ -43,6 +43,7 @@
     "ppppp-caps": "github:staltz/ppppp-caps",
     "ppppp-goals": "github:staltz/ppppp-goals",
     "ppppp-keypair": "github:staltz/ppppp-keypair",
+    "ppppp-set": "github:staltz/ppppp-set",
     "prettier": "^2.6.2",
     "pretty-quick": "^3.1.3",
     "rimraf": "^4.4.0",

test/set-sync.test.js

@@ -0,0 +1,161 @@
+const test = require('node:test')
+const assert = require('node:assert')
+const p = require('node:util').promisify
+const Keypair = require('ppppp-keypair')
+const MsgV3 = require('ppppp-db/msg-v3')
+const { createPeer } = require('./util')
+
+const aliceKeypair = Keypair.generate('ed25519', 'alice')
+
+function getItems(arr) {
+  return arr
+    .filter((msg) => msg.metadata.domain === 'set_v1__follows')
+    .map((msg) => msg.data)
+    .filter((data) => !!data)
+    .map((data) => data.add?.[0] ?? '-' + data.del?.[0])
+}
+
+//
+// R-?-?-?-?-o-o
+//   \
+//    o
+//
+// where "o" is a set update and "?" is a ghost
+test('sync goal=set with ghostSpan=2', async (t) => {
+  const SPAN = 5
+  const alice = createPeer({
+    name: 'alice',
+    keypair: aliceKeypair,
+    set: { ghostSpan: SPAN },
+  })
+  const bob = createPeer({ name: 'bob' })
+
+  await alice.db.loaded()
+  await bob.db.loaded()
+
+  // Alice sets up an account and a set
+  const aliceID = await p(alice.db.account.create)({
+    domain: 'account',
+    _nonce: 'alice',
+  })
+  await p(alice.set.load)(aliceID)
+  const aliceAccountRoot = alice.db.get(aliceID)
+
+  // Bob knows Alice
+  await p(bob.db.add)(aliceAccountRoot, aliceID)
+
+  // Alice constructs a set
+  await p(alice.set.add)('follows', 'alice')
+  await p(alice.set.add)('follows', 'bob')
+  await p(alice.set.del)('follows', 'alice')
+  await p(alice.set.del)('follows', 'bob')
+  await p(alice.set.add)('follows', 'Alice')
+  await p(alice.set.add)('follows', 'Bob')
+  let moot
+  let rec1
+  let rec2
+  let rec3
+  let rec4
+  let rec5
+  let rec6
+  for (const rec of alice.db.records()) {
+    if (rec.msg.metadata.dataSize === 0) moot = rec
+    if (rec.msg.data?.add?.[0] === 'alice') rec1 = rec
+    if (rec.msg.data?.add?.[0] === 'bob') rec2 = rec
+    if (rec.msg.data?.del?.[0] === 'alice') rec3 = rec
+    if (rec.msg.data?.del?.[0] === 'bob') rec4 = rec
+    if (rec.msg.data?.add?.[0] === 'Alice') rec5 = rec
+    if (rec.msg.data?.add?.[0] === 'Bob') rec6 = rec
+  }
+
+console.log('moot', moot.id);
+console.log('msg1', rec1.id);
+console.log('msg2', rec2.id);
+console.log('msg3', rec3.id);
+console.log('msg4', rec4.id);
+console.log('msg5', rec5.id);
+console.log('msg6', rec6.id);
+
+  // Bob knows the whole set
+  await p(bob.db.add)(moot.msg, moot.id)
+  await p(bob.db.add)(rec1.msg, moot.id)
+  await p(bob.db.add)(rec2.msg, moot.id)
+  await p(bob.db.add)(rec3.msg, moot.id)
+  await p(bob.db.add)(rec4.msg, moot.id)
+  await p(bob.db.add)(rec5.msg, moot.id)
+  await p(bob.db.add)(rec6.msg, moot.id)
+
+  // Bob knows a branched off msg that Alice doesn't know
+  {
+    const tangle = new MsgV3.Tangle(moot.id)
+    tangle.add(moot.id, moot.msg)
+    tangle.add(rec1.id, rec1.msg)
+    const msg = MsgV3.create({
+      keypair: aliceKeypair,
+      domain: 'set_v1__follows',
+      account: aliceID,
+      accountTips: [aliceID],
+      data: { add: ['Carol'], del: [], supersedes: [] },
+      tangles: {
+        [moot.id]: tangle,
+      },
+    })
+    await p(bob.db.add)(msg, moot.id)
+  }
+
+  // Simulate Alice garbage collecting part of the set
+  {
+    const itemRoots = alice.set._getItemRoots('follows')
+    assert.deepEqual(itemRoots, { Alice: [rec5.id], Bob: [rec6.id] })
+    const tangle = alice.db.getTangle(alice.set.getFeedID('follows'))
+    const { deletables, erasables } = tangle.getDeletablesAndErasables(rec5.id)
+    assert.equal(deletables.size, 2)
+    assert.equal(erasables.size, 3)
+    assert.ok(deletables.has(rec1.id))
+    assert.ok(deletables.has(rec2.id))
+    assert.ok(erasables.has(moot.id))
+    assert.ok(erasables.has(rec3.id))
+    assert.ok(erasables.has(rec4.id))
+
+    for (const msgID of deletables) {
+      await p(alice.db.ghosts.add)({ msgID, tangleID: moot.id, span: SPAN })
+      await p(alice.db.del)(msgID)
+    }
+    for (const msgID of erasables) {
+      if (msgID === moot.id) continue
+      await p(alice.db.erase)(msgID)
+    }
+  }
+
+  // Assert situation at Alice before tangleSync
+  {
+    const arr = getItems([...alice.db.msgs()])
+    console.log(arr)
+    assert.deepEqual(arr, ['Alice', 'Bob'], 'alice has Alice+Bob set')
+  }
+  assert.deepEqual(alice.db.ghosts.get(moot.id), [rec1.id, rec2.id])
+
+  // Trigger tangleSync
+  alice.goals.set(moot.id, 'set')
+  bob.goals.set(moot.id, 'set')
+  const remoteAlice = await p(bob.connect)(alice.getAddress())
+  assert('bob connected to alice')
+  bob.tangleSync.initiate()
+  await p(setTimeout)(1000)
+  assert('tangleSync!')
+
+  // Assert situation at Alice after tangleSync: she got the branched off msg
+  {
+    const arr = getItems([...alice.db.msgs()])
+    assert.deepEqual(
+      arr,
+      ['Alice', 'Bob', 'Carol'],
+      'alice has Alice+Bob+Carol set'
+    )
+  }
+  assert.deepEqual(alice.db.ghosts.get(moot.id), [rec2.id])
+
+  await p(remoteAlice.close)(true)
+  await p(alice.close)(true)
+  await p(bob.close)(true)
+})

test/util.js

@@ -20,6 +20,7 @@ function createPeer(opts) {
     .use(require('secret-handshake-ext/secret-stack'))
     .use(require('ppppp-db'))
     .use(require('ppppp-dict'))
+    .use(require('ppppp-set'))
     .use(require('ppppp-goals'))
     .use(require('ssb-box'))
     .use(require('../lib'))