diff --git a/lib/index.js b/lib/index.js index 325c3a7..82ab79f 100644 --- a/lib/index.js +++ b/lib/index.js @@ -86,9 +86,41 @@ function initConductor(peer, config) { assertGCPlugin(peer) assertSyncPlugin(peer) - const ESTIMATE_TOTAL_GHOST_BYTES = 1024 * 1024 // 1 MB + /** + * How many bytes does a single msg ID take up + */ const MSG_ID_BYTES = MsgV3.getMootID('dummy', 'dummy').length + /** + * How many bytes does an average msg take up + */ + const ESTIMATE_MSG_SIZE = 600 // 600 bytes + + /** + * How many bytes should we budget for ghost msg IDs in total in the database + */ + const ESTIMATE_TOTAL_GHOST_BYTES = 1024 * 1024 // 1 MB + + /** + * How many msgs does the average 'follow' Set feed contain + */ + const ESTIMATE_FOLLOW_FEED_SIZE = 300 + + /** + * How many msgs does the average 'block' Set feed contain + */ + const ESTIMATE_BLOCK_FEED_SIZE = 30 + + /** + * How many msgs does the average unknown feed contain + */ + const ESTIMATE_FEED_SIZE = 100 + + const MIN_MAXBYTES = 1024 // 1 kB + const MIN_RECOMMENDED_MAXBYTES = 32 * 1024 * 1024 // 32 MB + const GOOD_RECOMMENDED_MAXBYTES = 64 * 1024 * 1024 // 64 MB + const MAX_RECOMMENDED_MAXBYTES = 100 * 1024 * 1024 // 100 MB + const debug = makeDebug('ppppp:conductor') /** @@ -104,6 +136,57 @@ function initConductor(peer, config) { return count } + /** + * @param {Rule} rule + */ + function getRealisticCount(rule) { + assertGoalsPlugin(peer) + const [, goalDSL] = parseRule(rule) + const { count } = peer.goals.parse(goalDSL) + const realisticCount = isFinite(count) + ? count + : Math.min(count, ESTIMATE_FEED_SIZE) + return realisticCount + } + + /** + * Parses input rules. If goals are too big for maxBytes budget, scale down + * goals. + * + * @param {[Array, Array]} rules + * @param {number} numFollowed + * @param {number} maxBytes + * @returns {[Array, Array]} + */ + function validateRules(rules, numFollowed, maxBytes) { + assertGoalsPlugin(peer) + + const [myRules, theirRules] = rules + + let estimateMsgCount = + (1 + numFollowed) * ESTIMATE_FOLLOW_FEED_SIZE + ESTIMATE_BLOCK_FEED_SIZE + for (const rule of myRules) { + estimateMsgCount += getRealisticCount(rule) + } + for (const rule of theirRules) { + estimateMsgCount += numFollowed * getRealisticCount(rule) + } + + const estimateBytesUsed = estimateMsgCount * ESTIMATE_MSG_SIZE + const factor = maxBytes / estimateBytesUsed + if (estimateBytesUsed > maxBytes) { + if (maxBytes < MIN_RECOMMENDED_MAXBYTES) { + // prettier-ignore + debug('WARNING. maxBytes is in practice too small, we recommend at least %s bytes, ideally %s bytes, and at most %s bytes', MIN_RECOMMENDED_MAXBYTES, GOOD_RECOMMENDED_MAXBYTES, MAX_RECOMMENDED_MAXBYTES) + } else { + // prettier-ignore + debug('WARNING. maxBytes might be easily surpassed, you should downscale rules to %s%', (factor*100).toFixed(0)) + } + } + + return [myRules, theirRules] + } + /** * Set replication goals for various tangles of an account: * - Account tangle @@ -183,13 +266,20 @@ function initConductor(peer, config) { assertGCPlugin(peer) assertSyncPlugin(peer) - const [myRules, theirRules] = rules + if (maxBytes < MIN_MAXBYTES) { + // prettier-ignore + throw new Error(`ppppp-conductor maxBytes must be at least ${MIN_MAXBYTES} bytes, got ${maxBytes}`) + } + if (maxBytes > MAX_RECOMMENDED_MAXBYTES) { + debug('WARNING. maxBytes is too big, we recommend at most %s bytes', MAX_RECOMMENDED_MAXBYTES) + } - // TODO: If goals are too big for maxBytes budget, scale down goals + const followedAccounts = peer.set.values('follow') + const numFollowed = followedAccounts.length + const [myRules, theirRules] = validateRules(rules, numFollowed, maxBytes) // Set up goals for my account and each account I follow setupAccountGoals(myID, myRules) - const followedAccounts = peer.set.values('follow') for (const theirID of followedAccounts) { setupAccountGoals(theirID, theirRules) } @@ -210,7 +300,7 @@ function initConductor(peer, config) { // Figure out ghost span for each account const totalGhostableFeeds = countGhostableFeeds(myRules) + - followedAccounts.length * countGhostableFeeds(theirRules) + numFollowed * countGhostableFeeds(theirRules) const TOTAL_GHOSTS = ESTIMATE_TOTAL_GHOST_BYTES / MSG_ID_BYTES const ghostSpan = Math.round(TOTAL_GHOSTS / totalGhostableFeeds) peer.set.setGhostSpan(ghostSpan)