update with latest ppppp-db supporting compact

lib/index.js

@@ -58,14 +58,14 @@ function initGC(peer, config) {
   assertGoalsPlugin(peer)
   assertValidConfig(config)
 
-  // Constants
+  const MAX_LOG_BYTES = config.gc.maxLogBytes
-  const COMPACTION_INTERVAL = config.gc?.compactionInterval ?? 120e3
+
+  /** Number of records that match roughly 1% of the max log size */
+  const CHECKPOINT = Math.floor((MAX_LOG_BYTES * 0.01) / 500) // assuming 1 record = 500 bytes
 
   // State
   const debug = makeDebug('ppppp:gc')
-  let lastCompacted = Date.now()
   let stopMonitoringLogSize = /** @type {CallableFunction | null} */ (null)
-  let hasCompactionScheduled = false
   let hasCleanupScheduled = false
 
   /**
@@ -107,35 +107,13 @@ function initGC(peer, config) {
       // prettier-ignore
       if (err) debug('cleanup-per-purpose ended with an error %s', err.message ?? err)
       else debug('cleanup-per-purpose ended')
-      cb()
+      assertDBPlugin(peer)
+      peer.db.log.compact(cb)
     }
     if (waiting) done(whenEnded)
     else whenEnded()
   }
 
-  /**
-   * Compact the log (remove deleted records by filling in all the blanks).
-   * @private
-   * @param {number} waitPeriod
-   * @param {CB<void>} cb
-   */
-  function compact(waitPeriod, cb) {
-    assertDBPlugin(peer)
-    const log = peer.db._getLog() // TODO: use public API?
-    debug('compaction started')
-    /** @param {Error=} err */
-    function whenEnded(err) {
-      if (err) debug('compaction ended with an error %s', err.message ?? err)
-      else debug('compaction ended')
-      cb()
-    }
-    if (waitPeriod > 0) {
-      setTimeout(log.compact, waitPeriod, whenEnded)
-    } else {
-      log.compact(whenEnded)
-    }
-  }
-
   /**
    * Monitor the log size and schedule compaction and/or cleanup.
    */
@@ -143,24 +121,15 @@ function initGC(peer, config) {
     assertDBPlugin(peer)
     function checkLogSize() {
       assertDBPlugin(peer)
-      assertValidConfig(config)
+      peer.db.log.stats((err, stats) => {
-      peer.db.logStats((err, stats) => {
         if (err) return
-        const percentUsed = (stats.totalBytes / config.gc.maxLogBytes) * 100
+        const percentUsed = (stats.totalBytes / MAX_LOG_BYTES) * 100
-        const needsCompaction = stats.deletedBytes > 0
+        const percentDeleted = (stats.deletedBytes / stats.totalBytes) * 100
-
+        const needsCleanup = percentUsed > 80
-        // Schedule compaction
+        const needsCompaction = percentDeleted > 30
-        if (needsCompaction && !hasCompactionScheduled) {
-          const nextCompacted = lastCompacted + COMPACTION_INTERVAL
-          const waitPeriod = Math.max(0, nextCompacted - Date.now())
-          hasCompactionScheduled = true
-          compact(waitPeriod, () => {
-            hasCompactionScheduled = false
-          })
-        }
 
         // Schedule clean up
-        if (percentUsed > 80 && !hasCleanupScheduled) {
+        if ((needsCleanup || needsCompaction) && !hasCleanupScheduled) {
           hasCleanupScheduled = true
           cleanup(() => {
             hasCleanupScheduled = false
@@ -172,7 +141,7 @@ function initGC(peer, config) {
     let count = 0
     stopMonitoringLogSize = peer.db.onRecordAdded(() => {
       count += 1
-      if (count >= 1000) {
+      if (count >= CHECKPOINT) {
         count = 0
         checkLogSize()
       }

package.json

@@ -28,6 +28,7 @@
     "multicb": "^1.2.2"
   },
   "devDependencies": {
+    "@types/node": "18.x",
     "@types/debug": "4.1.9",
     "bs58": "^5.0.0",
     "c8": "7",

test/dict-ghosts.test.js

@@ -33,11 +33,11 @@ test('Dict ghosts', async (t) => {
 
   // Alice creates her own account
   const aliceID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     _nonce: 'alice',
   })
 
-  // Alice constructs adict
+  // Alice constructs a dict
   await p(alice.dict.load)(aliceID)
   await p(alice.dict.update)('profile', { name: 'alice' })
   await p(alice.dict.update)('profile', { age: 24 })
@@ -74,6 +74,12 @@ test('Dict ghosts', async (t) => {
   assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists')
   assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists')
 
+  assert.deepEqual(
+    await p(alice.db.log.stats)(),
+    { totalBytes: 3684, deletedBytes: 0 },
+    'log stats before'
+  )
+
   // Perform garbage collection
   alice.goals.set(aliceID, 'all')
   alice.goals.set(dictID, 'dict')
@@ -86,6 +92,12 @@ test('Dict ghosts', async (t) => {
     'alice has only field root msgs'
   )
 
+  assert.deepEqual(
+    await p(alice.db.log.stats)(),
+    { totalBytes: 2572, deletedBytes: 0 },
+    'log stats after'
+  )
+
   assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased')
   assert.ok(isDeleted(alice.db.get(msgID1)), 'msg1 deleted')
   assert.ok(isDeleted(alice.db.get(msgID2)), 'msg2 deleted') // ghost!

test/feed-decay.test.js

@@ -17,7 +17,7 @@ test('feed decay', async (t) => {
 
   // Alice creates her own account
   const aliceID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     _nonce: 'alice',
   })
 

test/feed-holes.test.js

@@ -17,7 +17,7 @@ test('feed holes', async (t) => {
 
   // Alice creates her own account
   const aliceID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     _nonce: 'alice',
   })
 

test/orphan-weave.test.js

@@ -21,18 +21,18 @@ test('orphan weave msgs', async (t) => {
 
   // Alice creates her own account
   const aliceID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     _nonce: 'alice',
   })
   // Alice creates Bob
   const bobID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     keypair: bobKeypair,
     _nonce: 'bob',
   })
   // Alice creates Carol
   const carolID = await p(alice.db.account.create)({
-    domain: 'account',
+    subdomain: 'account',
     keypair: carolKeypair,
     _nonce: 'carol',
   })