From bb7383aa6da3acd40f428b0c06ebab9b0042c0f2 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Thu, 21 Dec 2023 13:22:23 +0200
Subject: [PATCH] set ghostSpan

---
 lib/index.js              | 33 +++++++++++++++++++++++++---
 package.json              |  1 +
 test/follow-feeds.test.js | 45 +++++++++++++++++++++++++++++++++++++++
 test/util.js              |  1 +
 4 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/lib/index.js b/lib/index.js
index 05e42ec..8187706 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -5,14 +5,16 @@ const makeDebug = require('debug')
  * @typedef {ReturnType<import('ppppp-goals').init>} PPPPPGoal
  * @typedef {import('ppppp-goals').GoalDSL} GoalDSL
  * @typedef {ReturnType<import('ppppp-set').init>} PPPPPSet
+ * @typedef {ReturnType<import('ppppp-dict').init>} PPPPPDict
  * @typedef {ReturnType<import('ppppp-sync').init>} PPPPPSync
  * @typedef {ReturnType<import('ppppp-gc').init>} PPPPPGC
  * @typedef {`${string}@${GoalDSL}`} Rule
  * @typedef {[Array<Rule>, Array<Rule>]} Rules
  * @typedef {{
- *   db:PPPPPDB | null,
+ *   db: PPPPPDB | null,
  *   goals: PPPPPGoal | null,
  *   set: PPPPPSet | null,
+ *   dict: PPPPPDict | null,
  *   sync: PPPPPSync | null,
  *   gc: PPPPPGC | null,
  * }} UnknownPeer
@@ -83,8 +85,24 @@ function initConductor(peer, config) {
   assertGCPlugin(peer)
   assertSyncPlugin(peer)
 
+  const ESTIMATE_TOTAL_GHOST_BYTES = 1024 * 1024 // 1 MB
+  const ESTIMATE_MSG_ID_BYTES = 22 // 22 bytes per msg ID
+
   const debug = makeDebug('ppppp:conductor')
 
+  /**
+   * @param {Array<Rule>} rules
+   */
+  function countGhostableFeeds(rules) {
+    let count = 2 // 'follow' and 'block' Sets
+    for (const rule of rules) {
+      const [, goalDSL] = parseRule(rule)
+      if (goalDSL === 'dict') count++
+      else if (goalDSL === 'set') count++
+    }
+    return count
+  }
+
   /**
    * Set replication goals for various tangles of an account:
    * - Account tangle
@@ -167,10 +185,9 @@ function initConductor(peer, config) {
     const [myRules, theirRules] = rules
 
     // TODO: If goals are too big for maxBytes budget, scale down goals
-    // TODO: Figure out ghost spans for dicts and sets
 
+    // Set up goals for my account and each account I follow
     setupAccountGoals(myID, myRules)
-
     const followedAccounts = peer.set.values('follow')
     for (const theirID of followedAccounts) {
       setupAccountGoals(theirID, theirRules)
@@ -189,6 +206,16 @@ function initConductor(peer, config) {
       }
     })
 
+    // Figure out ghost span for each account
+    const totalGhostableFeeds =
+      countGhostableFeeds(myRules) +
+      followedAccounts.length * countGhostableFeeds(theirRules)
+    const TOTAL_GHOSTS = ESTIMATE_TOTAL_GHOST_BYTES / ESTIMATE_MSG_ID_BYTES
+    const ghostSpan = Math.round(TOTAL_GHOSTS / totalGhostableFeeds)
+    peer.set.setGhostSpan(ghostSpan)
+    peer.dict?.setGhostSpan(ghostSpan)
+
+    // Kick off garbage collection and synchronization
     peer.gc.start(maxBytes)
     peer.sync.start()
   }
diff --git a/package.json b/package.json
index 9abc548..aa8baa5 100644
--- a/package.json
+++ b/package.json
@@ -34,6 +34,7 @@
     "ppppp-caps": "github:staltz/ppppp-caps",
     "ppppp-db": "github:staltz/ppppp-db",
     "ppppp-set": "github:staltz/ppppp-set",
+    "ppppp-dict": "github:staltz/ppppp-dict",
     "ppppp-gc": "github:staltz/ppppp-gc",
     "ppppp-goals": "github:staltz/ppppp-goals",
     "ppppp-keypair": "github:staltz/ppppp-keypair",
diff --git a/test/follow-feeds.test.js b/test/follow-feeds.test.js
index 2257b5c..90524e5 100644
--- a/test/follow-feeds.test.js
+++ b/test/follow-feeds.test.js
@@ -350,3 +350,48 @@ test('GC recently-blocked accounts', async (t) => {
   await p(bob.close)(true)
   await p(carol.close)(true)
 })
+
+test('Set and Dict ghost spans', async (t) => {
+  // Alice
+  const alice = createPeer({ name: 'alice' })
+  await alice.db.loaded()
+  // Alice creates her own account
+  const aliceID = await p(alice.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'alice',
+  })
+  await p(alice.set.load)(aliceID)
+
+  // Bob
+  const bob = createPeer({ name: 'bob' })
+  await bob.db.loaded()
+  // Bob creates his own account
+  const bobID = await p(bob.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'bob',
+  })
+  await p(bob.set.load)(bobID)
+
+  // Carol
+  const carol = createPeer({ name: 'carol' })
+  await carol.db.loaded()
+  // Carol creates her own account
+  const carolID = await p(carol.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'carol',
+  })
+  await p(carol.set.load)(bobID)
+
+  // Alice follows Bob, but not Carol
+  assert(await p(alice.set.add)('follow', bobID), 'alice follows bob')
+
+  alice.conductor.start(aliceID, [['post@all'], ['post@all']], 4_000)
+  bob.conductor.start(bobID, [['post@all'], ['post@all']], 4_000)
+
+  assert.equal(alice.set.getGhostSpan(), 11916, 'alice set ghost span is 2')
+  assert.equal(alice.dict.getGhostSpan(), 11916, 'alice set ghost span is 2')
+
+  await p(alice.close)(true)
+  await p(bob.close)(true)
+  await p(carol.close)(true)
+})
diff --git a/test/util.js b/test/util.js
index 4347934..38bf27f 100644
--- a/test/util.js
+++ b/test/util.js
@@ -22,6 +22,7 @@ function createPeer(opts) {
     .use(require('ppppp-db'))
     .use(require('ssb-box'))
     .use(require('ppppp-set'))
+    .use(require('ppppp-dict'))
     .use(require('ppppp-goals'))
     .use(require('ppppp-sync'))
     .use(require('ppppp-gc'))