can replicate record tangles containing ghosts

lib/algorithm.js

@@ -1,5 +1,3 @@
-// @ts-ignore
-const multicb = require('multicb')
 const p = require('promisify-4loc')
 const { BloomFilter } = require('bloom-filters')
 const MsgV3 = require('ppppp-db/msg-v3')
@@ -7,6 +5,7 @@ const { EMPTY_RANGE, isEmptyRange, estimateMsgCount } = require('./range')
 
 /**
  * @typedef {ReturnType<import('ppppp-db').init>} PPPPPDB
+ * @typedef {ReturnType<import('ppppp-record').init>} PPPPPRecord
  * @typedef {import('ppppp-db/msg-v3').Msg} Msg
  * @typedef {import('ppppp-goals').Goal} Goal
  * @typedef {import('./range').Range} Range
@@ -22,11 +21,21 @@ function countIter(iter) {
   return count
 }
 
+/**
+ * @param {{ record: PPPPPRecord | null }} peer
+ * @returns {asserts peer is { record: PPPPPRecord }}
+ */
+function assertRecordPlugin(peer) {
+  if (!peer.record) {
+    throw new Error('tanglesync plugin requires ppppp-record plugin')
+  }
+}
+
 class Algorithm {
-  /** @type {{ db: PPPPPDB }} */
+  /** @type {{ db: PPPPPDB, record: PPPPPRecord | null }} */
   #peer
 
-  /** @param {{ db: PPPPPDB }} peer */
+  /** @param {{ db: PPPPPDB, record: PPPPPRecord | null }} peer */
   constructor(peer) {
     this.#peer = peer
   }
@@ -41,7 +50,7 @@ class Algorithm {
     let minDepth = Number.MAX_SAFE_INTEGER
     let maxDepth = 0
     for (const rec of this.#peer.db.records()) {
-      if (!rec?.msg?.data) continue
+      if (!rec?.msg?.data && rec.id !== rootID) continue
       const tangles = rec.msg.metadata.tangles
       if (rec.id === rootID) {
         minDepth = 0
@@ -83,6 +92,19 @@ class Algorithm {
     return [minWant, maxWant]
   }
 
+  /**
+   * @param {number} minGhostDepth
+   * @param {Range} remoteHaveRange
+   * @returns {Range}
+   */
+  #wantRecordRange(minGhostDepth, remoteHaveRange) {
+    const [minRemoteHave, maxRemoteHave] = remoteHaveRange
+    if (maxRemoteHave < minGhostDepth) return EMPTY_RANGE
+    const maxWant = maxRemoteHave
+    const minWant = Math.max(minGhostDepth, minRemoteHave)
+    return [minWant, maxWant]
+  }
+
   /**
    * @param {Range} localHave
    * @param {Range} remoteHave
@@ -97,6 +119,14 @@ class Algorithm {
       case 'all':
         return this.#wantAllRange(localHave, remoteHave)
 
+      case 'record':
+        assertRecordPlugin(this.#peer)
+        const minGhostDepth = this.#peer.record.minGhostDepth(goal.id)
+        return this.#wantRecordRange(minGhostDepth, remoteHave)
+
+      case 'set':
+        throw new Error('Not implemented') // TODO
+
       case 'newest':
         return this.#wantNewestRange(localHave, remoteHave, goal.count)
 
@@ -124,8 +154,14 @@ class Algorithm {
         filter.add('' + round + MsgV3.getMsgID(msg))
       }
     }
-    for (const msgId of extraIds) {
-      filter.add('' + round + msgId)
+    const ghosts = this.#peer.db.ghosts.get(rootID)
+    for (const ghostMsgID of ghosts) {
+      // No need to check depths because the `range` is by definition taking
+      // into account local ghost depths
+      filter.add('' + round + ghostMsgID)
+    }
+    for (const msgID of extraIds) {
+      filter.add('' + round + msgID)
     }
     return filter.saveAsJSON()
   }
@@ -208,7 +244,7 @@ class Algorithm {
           return false // the rootMsg is the only acceptable depth-zero msg
         }
         if (!msg.data) {
-          return depth < minWant
+          return depth <= maxWant
         } else {
           return minWant <= depth && depth <= maxWant
         }
@@ -234,15 +270,13 @@ class Algorithm {
     // TODO: Simulate adding this whole tangle, and check if it's valid
 
     // Add new messages
-    const doneAdding = multicb({ pluck: 1 })
     for (const msg of validNewMsgs) {
-      this.#peer.db.add(msg, rootID, doneAdding())
-    }
-    try {
-      await p(doneAdding)()
-    } catch (err) {
-      // TODO:
-      // debug('commit failed %o', err)
+      try {
+        await p(this.#peer.db.add)(msg, rootID) //, doneAdding())
+      } catch (err) {
+        // TODO:
+        // debug('commit failed %o', err)
+      }
     }
 
     if (goal.type === 'newest') {

test/record-sync.test.js

@@ -0,0 +1,147 @@
+const test = require('node:test')
+const assert = require('node:assert')
+const p = require('node:util').promisify
+const Keypair = require('ppppp-keypair')
+const MsgV3 = require('ppppp-db/msg-v3')
+const { createPeer } = require('./util')
+
+const aliceKeypair = Keypair.generate('ed25519', 'alice')
+
+//
+// R-?-?-o-o
+//   \
+//    o
+//
+// where "o" is a record update and "?" is a ghost
+test('sync goal=record with ghostSpan=2', async (t) => {
+  const SPAN = 5
+  const alice = createPeer({
+    name: 'alice',
+    keypair: aliceKeypair,
+    record: { ghostSpan: SPAN },
+  })
+  const bob = createPeer({ name: 'bob' })
+
+  await alice.db.loaded()
+  await bob.db.loaded()
+
+  // Alice sets up an account and a record
+  const aliceID = await p(alice.db.account.create)({
+    domain: 'account',
+    _nonce: 'alice',
+  })
+  await p(alice.record.load)(aliceID)
+  const aliceAccountRoot = alice.db.get(aliceID)
+
+  // Bob knows Alice
+  await p(bob.db.add)(aliceAccountRoot, aliceID)
+
+  // Alice constructs a record
+  await p(alice.record.update)('profile', { name: 'alice' })
+  await p(alice.record.update)('profile', { age: 24 })
+  await p(alice.record.update)('profile', { name: 'Alice' })
+  await p(alice.record.update)('profile', { age: 25 })
+  await p(alice.record.update)('profile', { name: 'ALICE' })
+  let moot
+  let rec1
+  let rec2
+  let rec3
+  let rec4
+  let rec5
+  for (const rec of alice.db.records()) {
+    if (rec.msg.metadata.dataSize === 0) moot = rec
+    if (rec.msg.data?.update?.name === 'alice') rec1 = rec
+    if (rec.msg.data?.update?.age === 24) rec2 = rec
+    if (rec.msg.data?.update?.name === 'Alice') rec3 = rec
+    if (rec.msg.data?.update?.age === 25) rec4 = rec
+    if (rec.msg.data?.update?.name === 'ALICE') rec5 = rec
+  }
+
+  // Bob knows the whole record
+  await p(bob.db.add)(moot.msg, moot.id)
+  await p(bob.db.add)(rec1.msg, moot.id)
+  await p(bob.db.add)(rec2.msg, moot.id)
+  await p(bob.db.add)(rec3.msg, moot.id)
+  await p(bob.db.add)(rec4.msg, moot.id)
+  await p(bob.db.add)(rec5.msg, moot.id)
+
+  // Bob knows a branched off msg that Alice doesn't know
+  {
+    const tangle = new MsgV3.Tangle(moot.id)
+    tangle.add(moot.id, moot.msg)
+    tangle.add(rec1.id, rec1.msg)
+    const msg = MsgV3.create({
+      keypair: aliceKeypair,
+      domain: 'record_v1__profile',
+      account: aliceID,
+      accountTips: [aliceID],
+      data: { update: { gender: 'w' }, supersedes: [] },
+      tangles: {
+        [moot.id]: tangle,
+      },
+    })
+    const recX = await p(bob.db.add)(msg, moot.id)
+    assert.equal(recX.id, 'Pd8e1aMN7yFWSr7yc1qycr', 'msg ID')
+  }
+
+  // Simulate Alice garbage collecting part of the record
+  {
+    const fieldRoots = alice.record._getFieldRoots('profile')
+    assert.deepEqual(fieldRoots.age, [rec4.id])
+    assert.deepEqual(fieldRoots.name, [rec5.id])
+    const tangle = alice.db.getTangle(alice.record.getFeedID('profile'))
+    const { deletables, erasables } = tangle.getDeletablesAndErasables(rec4.id)
+    assert.equal(deletables.size, 2)
+    assert.equal(erasables.size, 2)
+    assert.ok(deletables.has(rec1.id))
+    assert.ok(deletables.has(rec2.id))
+    assert.ok(erasables.has(rec3.id))
+    assert.ok(erasables.has(moot.id))
+
+    for (const msgID of deletables) {
+      await p(alice.db.ghosts.add)({ msgID, tangleID: moot.id, max: SPAN })
+      await p(alice.db.del)(msgID)
+    }
+    for (const msgID of erasables) {
+      if (msgID === moot.id) continue
+      await p(alice.db.erase)(msgID)
+    }
+  }
+
+  // Assert situation at Alice before tangleSync
+  {
+    const arr = [...alice.db.msgs()]
+      .map((msg) => msg.data?.update)
+      .filter((x) => !!x)
+      .map((x) => x.age ?? x.name ?? x.gender)
+    assert.deepEqual(arr, [25, 'ALICE'], 'alice has age+name record')
+  }
+  assert.deepEqual(alice.db.ghosts.get(moot.id), [rec1.id, rec2.id])
+
+  // Trigger tangleSync
+  alice.goals.set(moot.id, 'record')
+  bob.goals.set(moot.id, 'record')
+  const remoteAlice = await p(bob.connect)(alice.getAddress())
+  assert('bob connected to alice')
+  bob.tangleSync.initiate()
+  await p(setTimeout)(1000)
+  assert('tangleSync!')
+
+  // Assert situation at Alice before tangleSync: she got the branched off msg
+  {
+    const arr = [...alice.db.msgs()]
+      .map((msg) => msg.data?.update)
+      .filter((x) => !!x)
+      .map((x) => x.age ?? x.name ?? x.gender)
+    assert.deepEqual(
+      arr,
+      [25, 'ALICE', 'w'],
+      'alice has age+name+gender record'
+    )
+  }
+  assert.deepEqual(alice.db.ghosts.get(moot.id), [rec2.id])
+
+  await p(remoteAlice.close)(true)
+  await p(alice.close)(true)
+  await p(bob.close)(true)
+})

test/util.js

@@ -6,7 +6,8 @@ const Keypair = require('ppppp-keypair')
 
 function createPeer(opts) {
   if (opts.name) {
-    opts.path ??= path.join(os.tmpdir(), 'tanglesync-' + opts.name)
+    const tmp = os.tmpdir()
+    opts.path ??= path.join(tmp, `tanglesync-${opts.name}-${Date.now()}`)
     opts.keypair ??= Keypair.generate('ed25519', opts.name)
     opts.name = undefined
   }
@@ -18,6 +19,7 @@ function createPeer(opts) {
     .use(require('secret-stack/plugins/net'))
     .use(require('secret-handshake-ext/secret-stack'))
     .use(require('ppppp-db'))
+    .use(require('ppppp-record'))
     .use(require('ppppp-goals'))
     .use(require('ssb-box'))
     .use(require('../lib'))

test/want-range.test.js

@@ -1,9 +1,10 @@
 const test = require('node:test')
 const assert = require('node:assert')
-const p = require('node:util').promisify
 const Algorithm = require('../lib/algorithm')
 
-test('want-range for goal=newest-3', async (t) => {
+const EMPTY = [1, 0]
+
+test('want-range for goal=newest-3', (t) => {
   const algo = new Algorithm({ db: null })
   const goal = { type: 'newest', count: 3 }
 
@@ -12,11 +13,11 @@ test('want-range for goal=newest-3', async (t) => {
   assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [2, 4])
   assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [3, 4])
   assert.deepStrictEqual(algo.wantRange([1, 3], [4, 6], goal), [4, 6])
-  assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), [1, 0])
+  assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), EMPTY)
   assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7])
 })
 
-test('want-range for goal=all', async (t) => {
+test('want-range for goal=all', (t) => {
   const algo = new Algorithm({ db: null })
   const goal = { type: 'all' }
 
@@ -29,15 +30,15 @@ test('want-range for goal=all', async (t) => {
   assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7])
 })
 
-test('want-range for goal=record', async (t) => {
-  const algo = new Algorithm({ db: null })
+test('want-range for goal=record', (t) => {
+  const algo = new Algorithm({ db: null, record: { minGhostDepth: () => 3 } })
   const goal = { type: 'record' }
 
-  assert.deepStrictEqual(algo.wantRange([2, 4], [1, 3], goal), [2, 3])
-  assert.deepStrictEqual(algo.wantRange([2, 4], [1, 5], goal), [2, 5])
-  assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [2, 4])
-  assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [2, 4])
+  assert.deepStrictEqual(algo.wantRange([2, 4], [1, 3], goal), [3, 3])
+  assert.deepStrictEqual(algo.wantRange([2, 4], [1, 5], goal), [3, 5])
+  assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [3, 4])
+  assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [3, 4])
   assert.deepStrictEqual(algo.wantRange([1, 3], [4, 6], goal), [4, 6])
-  assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), [1, 0])
+  assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), [3, 3])
   assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7])
 })