From 7b1119795e2415a240168f0bec68ff363b6fa46a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Nov 2023 17:14:38 +0200 Subject: [PATCH] update with latest ppppp-db supporting compact --- lib/index.js | 57 +++++++++------------------------------ package.json | 1 + test/dict-ghosts.test.js | 16 +++++++++-- test/feed-decay.test.js | 2 +- test/feed-holes.test.js | 2 +- test/orphan-weave.test.js | 6 ++--- 6 files changed, 33 insertions(+), 51 deletions(-) diff --git a/lib/index.js b/lib/index.js index b382baa..42fe6fd 100644 --- a/lib/index.js +++ b/lib/index.js @@ -58,14 +58,14 @@ function initGC(peer, config) { assertGoalsPlugin(peer) assertValidConfig(config) - // Constants - const COMPACTION_INTERVAL = config.gc?.compactionInterval ?? 120e3 + const MAX_LOG_BYTES = config.gc.maxLogBytes + + /** Number of records that match roughly 1% of the max log size */ + const CHECKPOINT = Math.floor((MAX_LOG_BYTES * 0.01) / 500) // assuming 1 record = 500 bytes // State const debug = makeDebug('ppppp:gc') - let lastCompacted = Date.now() let stopMonitoringLogSize = /** @type {CallableFunction | null} */ (null) - let hasCompactionScheduled = false let hasCleanupScheduled = false /** @@ -107,35 +107,13 @@ function initGC(peer, config) { // prettier-ignore if (err) debug('cleanup-per-purpose ended with an error %s', err.message ?? err) else debug('cleanup-per-purpose ended') - cb() + assertDBPlugin(peer) + peer.db.log.compact(cb) } if (waiting) done(whenEnded) else whenEnded() } - /** - * Compact the log (remove deleted records by filling in all the blanks). - * @private - * @param {number} waitPeriod - * @param {CB} cb - */ - function compact(waitPeriod, cb) { - assertDBPlugin(peer) - const log = peer.db._getLog() // TODO: use public API? - debug('compaction started') - /** @param {Error=} err */ - function whenEnded(err) { - if (err) debug('compaction ended with an error %s', err.message ?? err) - else debug('compaction ended') - cb() - } - if (waitPeriod > 0) { - setTimeout(log.compact, waitPeriod, whenEnded) - } else { - log.compact(whenEnded) - } - } - /** * Monitor the log size and schedule compaction and/or cleanup. */ @@ -143,24 +121,15 @@ function initGC(peer, config) { assertDBPlugin(peer) function checkLogSize() { assertDBPlugin(peer) - assertValidConfig(config) - peer.db.logStats((err, stats) => { + peer.db.log.stats((err, stats) => { if (err) return - const percentUsed = (stats.totalBytes / config.gc.maxLogBytes) * 100 - const needsCompaction = stats.deletedBytes > 0 - - // Schedule compaction - if (needsCompaction && !hasCompactionScheduled) { - const nextCompacted = lastCompacted + COMPACTION_INTERVAL - const waitPeriod = Math.max(0, nextCompacted - Date.now()) - hasCompactionScheduled = true - compact(waitPeriod, () => { - hasCompactionScheduled = false - }) - } + const percentUsed = (stats.totalBytes / MAX_LOG_BYTES) * 100 + const percentDeleted = (stats.deletedBytes / stats.totalBytes) * 100 + const needsCleanup = percentUsed > 80 + const needsCompaction = percentDeleted > 30 // Schedule clean up - if (percentUsed > 80 && !hasCleanupScheduled) { + if ((needsCleanup || needsCompaction) && !hasCleanupScheduled) { hasCleanupScheduled = true cleanup(() => { hasCleanupScheduled = false @@ -172,7 +141,7 @@ function initGC(peer, config) { let count = 0 stopMonitoringLogSize = peer.db.onRecordAdded(() => { count += 1 - if (count >= 1000) { + if (count >= CHECKPOINT) { count = 0 checkLogSize() } diff --git a/package.json b/package.json index f8c7013..ce927b4 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "multicb": "^1.2.2" }, "devDependencies": { + "@types/node": "18.x", "@types/debug": "4.1.9", "bs58": "^5.0.0", "c8": "7", diff --git a/test/dict-ghosts.test.js b/test/dict-ghosts.test.js index d437f5f..bfd88fb 100644 --- a/test/dict-ghosts.test.js +++ b/test/dict-ghosts.test.js @@ -33,11 +33,11 @@ test('Dict ghosts', async (t) => { // Alice creates her own account const aliceID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', _nonce: 'alice', }) - // Alice constructs adict + // Alice constructs a dict await p(alice.dict.load)(aliceID) await p(alice.dict.update)('profile', { name: 'alice' }) await p(alice.dict.update)('profile', { age: 24 }) @@ -74,6 +74,12 @@ test('Dict ghosts', async (t) => { assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists') assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists') + assert.deepEqual( + await p(alice.db.log.stats)(), + { totalBytes: 3684, deletedBytes: 0 }, + 'log stats before' + ) + // Perform garbage collection alice.goals.set(aliceID, 'all') alice.goals.set(dictID, 'dict') @@ -86,6 +92,12 @@ test('Dict ghosts', async (t) => { 'alice has only field root msgs' ) + assert.deepEqual( + await p(alice.db.log.stats)(), + { totalBytes: 2572, deletedBytes: 0 }, + 'log stats after' + ) + assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased') assert.ok(isDeleted(alice.db.get(msgID1)), 'msg1 deleted') assert.ok(isDeleted(alice.db.get(msgID2)), 'msg2 deleted') // ghost! diff --git a/test/feed-decay.test.js b/test/feed-decay.test.js index 15d4e1d..ca6ec89 100644 --- a/test/feed-decay.test.js +++ b/test/feed-decay.test.js @@ -17,7 +17,7 @@ test('feed decay', async (t) => { // Alice creates her own account const aliceID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', _nonce: 'alice', }) diff --git a/test/feed-holes.test.js b/test/feed-holes.test.js index d1f95ef..d7cf446 100644 --- a/test/feed-holes.test.js +++ b/test/feed-holes.test.js @@ -17,7 +17,7 @@ test('feed holes', async (t) => { // Alice creates her own account const aliceID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', _nonce: 'alice', }) diff --git a/test/orphan-weave.test.js b/test/orphan-weave.test.js index 2f80c50..b19a842 100644 --- a/test/orphan-weave.test.js +++ b/test/orphan-weave.test.js @@ -21,18 +21,18 @@ test('orphan weave msgs', async (t) => { // Alice creates her own account const aliceID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', _nonce: 'alice', }) // Alice creates Bob const bobID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', keypair: bobKeypair, _nonce: 'bob', }) // Alice creates Carol const carolID = await p(alice.db.account.create)({ - domain: 'account', + subdomain: 'account', keypair: carolKeypair, _nonce: 'carol', })