recommend good maxBytes

lib/index.js

@@ -86,9 +86,41 @@ function initConductor(peer, config) {
   assertGCPlugin(peer)
   assertSyncPlugin(peer)
 
-  const ESTIMATE_TOTAL_GHOST_BYTES = 1024 * 1024 // 1 MB
+  /**
+   * How many bytes does a single msg ID take up
+   */
   const MSG_ID_BYTES = MsgV3.getMootID('dummy', 'dummy').length
 
+  /**
+   * How many bytes does an average msg take up
+   */
+  const ESTIMATE_MSG_SIZE = 600 // 600 bytes
+
+  /**
+   * How many bytes should we budget for ghost msg IDs in total in the database
+   */
+  const ESTIMATE_TOTAL_GHOST_BYTES = 1024 * 1024 // 1 MB
+
+  /**
+   * How many msgs does the average 'follow' Set feed contain
+   */
+  const ESTIMATE_FOLLOW_FEED_SIZE = 300
+
+  /**
+   * How many msgs does the average 'block' Set feed contain
+   */
+  const ESTIMATE_BLOCK_FEED_SIZE = 30
+
+  /**
+   * How many msgs does the average unknown feed contain
+   */
+  const ESTIMATE_FEED_SIZE = 100
+
+  const MIN_MAXBYTES = 1024 // 1 kB
+  const MIN_RECOMMENDED_MAXBYTES = 32 * 1024 * 1024 // 32 MB
+  const GOOD_RECOMMENDED_MAXBYTES = 64 * 1024 * 1024 // 64 MB
+  const MAX_RECOMMENDED_MAXBYTES = 100 * 1024 * 1024 // 100 MB
+
   const debug = makeDebug('ppppp:conductor')
 
   /**
@@ -104,6 +136,57 @@ function initConductor(peer, config) {
     return count
   }
 
+  /**
+   * @param {Rule} rule
+   */
+  function getRealisticCount(rule) {
+    assertGoalsPlugin(peer)
+    const [, goalDSL] = parseRule(rule)
+    const { count } = peer.goals.parse(goalDSL)
+    const realisticCount = isFinite(count)
+      ? count
+      : Math.min(count, ESTIMATE_FEED_SIZE)
+    return realisticCount
+  }
+
+  /**
+   * Parses input rules. If goals are too big for maxBytes budget, scale down
+   * goals.
+   *
+   * @param {[Array<Rule>, Array<Rule>]} rules
+   * @param {number} numFollowed
+   * @param {number} maxBytes
+   * @returns {[Array<Rule>, Array<Rule>]}
+   */
+  function validateRules(rules, numFollowed, maxBytes) {
+    assertGoalsPlugin(peer)
+
+    const [myRules, theirRules] = rules
+
+    let estimateMsgCount =
+      (1 + numFollowed) * ESTIMATE_FOLLOW_FEED_SIZE + ESTIMATE_BLOCK_FEED_SIZE
+    for (const rule of myRules) {
+      estimateMsgCount += getRealisticCount(rule)
+    }
+    for (const rule of theirRules) {
+      estimateMsgCount += numFollowed * getRealisticCount(rule)
+    }
+
+    const estimateBytesUsed = estimateMsgCount * ESTIMATE_MSG_SIZE
+    const factor = maxBytes / estimateBytesUsed
+    if (estimateBytesUsed > maxBytes) {
+      if (maxBytes < MIN_RECOMMENDED_MAXBYTES) {
+        // prettier-ignore
+        debug('WARNING. maxBytes is in practice too small, we recommend at least %s bytes, ideally %s bytes, and at most %s bytes', MIN_RECOMMENDED_MAXBYTES, GOOD_RECOMMENDED_MAXBYTES, MAX_RECOMMENDED_MAXBYTES)
+      } else {
+        // prettier-ignore
+        debug('WARNING. maxBytes might be easily surpassed, you should downscale rules to %s%', (factor*100).toFixed(0))
+      }
+    }
+
+    return [myRules, theirRules]
+  }
+
   /**
    * Set replication goals for various tangles of an account:
    * - Account tangle
@@ -183,13 +266,20 @@ function initConductor(peer, config) {
     assertGCPlugin(peer)
     assertSyncPlugin(peer)
 
-    const [myRules, theirRules] = rules
+    if (maxBytes < MIN_MAXBYTES) {
+      // prettier-ignore
+      throw new Error(`ppppp-conductor maxBytes must be at least ${MIN_MAXBYTES} bytes, got ${maxBytes}`)
+    }
+    if (maxBytes > MAX_RECOMMENDED_MAXBYTES) {
+      debug('WARNING. maxBytes is too big, we recommend at most %s bytes', MAX_RECOMMENDED_MAXBYTES)
+    }
 
-    // TODO: If goals are too big for maxBytes budget, scale down goals
+    const followedAccounts = peer.set.values('follow')
+    const numFollowed = followedAccounts.length
+    const [myRules, theirRules] = validateRules(rules, numFollowed, maxBytes)
 
     // Set up goals for my account and each account I follow
     setupAccountGoals(myID, myRules)
-    const followedAccounts = peer.set.values('follow')
     for (const theirID of followedAccounts) {
       setupAccountGoals(theirID, theirRules)
     }
@@ -210,7 +300,7 @@ function initConductor(peer, config) {
     // Figure out ghost span for each account
     const totalGhostableFeeds =
       countGhostableFeeds(myRules) +
-      followedAccounts.length * countGhostableFeeds(theirRules)
+      numFollowed * countGhostableFeeds(theirRules)
     const TOTAL_GHOSTS = ESTIMATE_TOTAL_GHOST_BYTES / MSG_ID_BYTES
     const ghostSpan = Math.round(TOTAL_GHOSTS / totalGhostableFeeds)
     peer.set.setGhostSpan(ghostSpan)