diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml deleted file mode 100644 index b3558e2..0000000 --- a/.github/workflows/node.js.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 10 - - strategy: - matrix: - node-version: [18.x, 20.x] - - steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: npm test diff --git a/.woodpecker.yaml b/.woodpecker.yaml new file mode 100644 index 0000000..d254f8d --- /dev/null +++ b/.woodpecker.yaml @@ -0,0 +1,13 @@ +matrix: + NODE_VERSION: + - 18 + - 20 + +steps: + test: + when: + event: [push] + image: node:${NODE_VERSION} + commands: + - npm install + - npm test \ No newline at end of file diff --git a/README.md b/README.md index 76f669d..b67c63d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -**Work in progress** +# pzp-gc ## Installation -We're not on npm yet. In your package.json, include this as - ```js -"ppppp-gc": "github:staltz/ppppp-gc" +npm install pzp-gc ``` diff --git a/lib/index.js b/lib/index.js index bde88da..77b676c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,9 +2,20 @@ const multicb = require('multicb') const makeDebug = require('debug') +// @ts-ignore +const p = (fn) => (...args) => { + return new Promise((res, rej) => { + // @ts-ignore + fn(...args, (err, val) => { + if (err) return rej(err) + return res(val) + }) + }) +} + /** - * @typedef {ReturnType} PPPPPDB - * @typedef {ReturnType} PPPPPGoal + * @typedef {ReturnType} pzpDB + * @typedef {ReturnType} pzpGoal * @typedef {{ * gc: { * maxLogBytes: number @@ -23,12 +34,12 @@ const makeDebug = require('debug') */ /** - * @param {{ db: PPPPPDB, goals: PPPPPGoal }} peer + * @param {{ db: pzpDB, goals: pzpGoal }} peer * @param {Config} config */ function initGC(peer, config) { // State - const debug = makeDebug('ppppp:gc') + const debug = makeDebug('pzp:gc') let stopMonitoringLogSize = /** @type {CallableFunction | null} */ (null) let hasCleanupScheduled = false @@ -47,77 +58,79 @@ function initGC(peer, config) { /** * Deletes messages that don't correspond to any goal. * @private - * @param {CB} cb + * @return {Promise} */ - function cleanup(cb) { - debug('Cleanup started') - const startTime = Date.now() - const done = multicb({ pluck: 1 }) + async function cleanup() { + return new Promise(async (res, rej) => { + debug('Cleanup started') + const startTime = Date.now() + const done = multicb({ pluck: 1 }) - /** - * @param {string} errExplanation - */ - function makeRecCB(errExplanation) { - const cb = done() - return (/**@type {Error=}*/ err) => { - if (err) debug('%s: %s', errExplanation, flattenCauseChain(err)) - cb() + /** + * @param {string} errExplanation + */ + function makeRecCB(errExplanation) { + const cb = done() + return (/**@type {Error=}*/ err) => { + if (err) debug('%s: %s', errExplanation, flattenCauseChain(err)) + cb() + } } - } - let waiting = false - for (const rec of peer.db.records()) { - if (!rec.msg) continue - const { id: msgID, msg } = rec - const [purpose, details] = peer.goals.getMsgPurpose(msgID, msg) - switch (purpose) { - case 'goal': { - continue // don't cleanup - } - case 'none': { - const recCB = makeRecCB('Failed to delete msg when cleaning up') - debug('Deleting msg %s with purpose=none', msgID) - peer.db.del(msgID, recCB) - waiting = true - continue - } - case 'ghost': { - const { tangleID, span } = details - const recCB = makeRecCB('Failed to delete ghost msg when cleaning up') - // TODO: Could one msg be a ghostable in MANY tangles? Or just one? - debug('Deleting and ghosting msg %s with purpose=ghost', msgID) - peer.db.ghosts.add({ tangleID, msgID, span }, (err) => { - if (err) return recCB(err) + let waiting = false + for await (const rec of peer.db.records()) { + if (!rec.msg) continue + const { id: msgID, msg } = rec + const [purpose, details] = await p(peer.goals.getMsgPurpose)(msgID, msg) + switch (purpose) { + case 'goal': { + continue // don't cleanup + } + case 'none': { + const recCB = makeRecCB('Failed to delete msg when cleaning up') + debug('Deleting msg %s with purpose=none', msgID) peer.db.del(msgID, recCB) - }) - waiting = true - continue - } - case 'trail': { - if (!msg.data) continue // it's already erased - const recCB = makeRecCB('Failed to erase trail msg when cleaning up') - debug('Erasing msg %s with purpose=trail', msgID) - peer.db.erase(msgID, recCB) - waiting = true - continue - } - default: { - cb(new Error('Unreachable')) - return + waiting = true + continue + } + case 'ghost': { + const { tangleID, span } = details + const recCB = makeRecCB('Failed to delete ghost msg when cleaning up') + // TODO: Could one msg be a ghostable in MANY tangles? Or just one? + debug('Deleting and ghosting msg %s with purpose=ghost', msgID) + peer.db.ghosts.add({ tangleID, msgID, span }, (err) => { + if (err) return recCB(err) + peer.db.del(msgID, recCB) + }) + waiting = true + continue + } + case 'trail': { + if (!msg.data) continue // it's already erased + const recCB = makeRecCB('Failed to erase trail msg when cleaning up') + debug('Erasing msg %s with purpose=trail', msgID) + peer.db.erase(msgID, recCB) + waiting = true + continue + } + default: { + rej(new Error('Unreachable')) + return + } } } - } - if (waiting) done(whenEnded) - else whenEnded() + if (waiting) done(whenEnded) + else whenEnded() - /** @param {Error=} err */ - function whenEnded(err) { - const duration = Date.now() - startTime - if (err) debug('Cleanup ended with an error %s', err.message ?? err) - else debug('Cleanup completed in %sms', duration) - cb() - } + /** @param {Error=} err */ + function whenEnded(err) { + const duration = Date.now() - startTime + if (err) debug('Cleanup ended with an error %s', err.message ?? err) + else debug('Cleanup completed in %sms', duration) + res() + } + }) } /** @@ -203,7 +216,7 @@ function initGC(peer, config) { if (needsCompaction) reportCompactionNeed(percentDeleted, stats) hasCleanupScheduled = true if (needsCleanup) { - cleanup(() => { + cleanup().finally(() => { compact(() => { hasCleanupScheduled = false }) @@ -253,7 +266,7 @@ function initGC(peer, config) { */ function forceImmediately(cb) { debug('Force clean and compact immediately') - cleanup(() => { + cleanup().finally(() => { compact(cb) }) } diff --git a/package.json b/package.json index d008adf..654ee89 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { - "name": "ppppp-gc", - "version": "1.0.0", - "description": "PPPPP garbage collector", + "name": "pzp-gc", + "version": "0.0.1", + "description": "PZP garbage collector", "author": "Andre Staltz ", "license": "MIT", - "homepage": "https://github.com/staltz/ppppp-gc", + "homepage": "https://codeberg.org/pzp/pzp-gc", "repository": { "type": "git", - "url": "git@github.com:staltz/ppppp-gc.git" + "url": "git@codeberg.org:pzp/pzp-gc.git" }, "main": "index.js", "files": [ @@ -31,12 +31,12 @@ "@types/debug": "4.1.9", "bs58": "^5.0.0", "c8": "7", - "ppppp-caps": "github:staltz/ppppp-caps#93fa810b9a40b78aef4872d4c2a8412cccb52929", - "ppppp-db": "github:staltz/ppppp-db#667b33779d98aff12a9b0cd2d7c80469a95cd04e", - "ppppp-dict": "github:staltz/ppppp-dict#6f0ff4e3383a8c18b766949f6db9b51460ecb640", - "ppppp-goals": "github:staltz/ppppp-goals#f862c2de624649906a4375711f3813db3b94a2ca", - "ppppp-keypair": "github:staltz/ppppp-keypair#61ef4420578f450dc2cc7b1efc1c5a691a871c74", - "ppppp-set": "github:staltz/ppppp-set#8983ba29f03db95a76b4bd9a55aa4392b350fdbb", + "pzp-caps": "^1.0.0", + "pzp-db": "^1.0.1", + "pzp-dict": "^1.0.0", + "pzp-goals": "^1.0.0", + "pzp-keypair": "^1.0.0", + "pzp-set": "^1.0.0", "prettier": "^2.6.2", "pretty-quick": "^3.1.3", "rimraf": "^4.4.0", diff --git a/test/dict-ghosts.test.js b/test/dict-ghosts.test.js index c18ba7e..3a24007 100644 --- a/test/dict-ghosts.test.js +++ b/test/dict-ghosts.test.js @@ -51,7 +51,7 @@ test('Dict ghosts', async (t) => { let msgID3 let msgID4 let msgID5 - for (const rec of alice.db.records()) { + for await (const rec of alice.db.records()) { if (rec.msg.metadata.dataSize === 0) mootID = rec.id if (rec.msg.data?.update?.name === 'alice') msgID1 = rec.id if (rec.msg.data?.update?.age === 24) msgID2 = rec.id @@ -60,18 +60,22 @@ test('Dict ghosts', async (t) => { if (rec.msg.data?.update?.name === 'ALICE') msgID5 = rec.id } + const msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } // Assert situation before GC assert.deepEqual( - getFields([...alice.db.msgs()]), + getFields(msgs), ['alice', 24, 'Alice', 25, 'ALICE'], 'has all dict msgs' ) - assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased') - assert.ok(isPresent(alice.db.get(msgID1)), 'msg1 exists') - assert.ok(isPresent(alice.db.get(msgID2)), 'msg2 exists') - assert.ok(isPresent(alice.db.get(msgID3)), 'msg3 exists') - assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists') - assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists') + assert.ok(isErased(await p(alice.db.get)(mootID)), 'moot by def erased') + assert.ok(isPresent(await p(alice.db.get)(msgID1)), 'msg1 exists') + assert.ok(isPresent(await p(alice.db.get)(msgID2)), 'msg2 exists') + assert.ok(isPresent(await p(alice.db.get)(msgID3)), 'msg3 exists') + assert.ok(isPresent(await p(alice.db.get)(msgID4)), 'msg4 exists') + assert.ok(isPresent(await p(alice.db.get)(msgID5)), 'msg5 exists') assert.deepEqual( await p(alice.db.log.stats)(), @@ -84,9 +88,13 @@ test('Dict ghosts', async (t) => { alice.goals.set(dictID, 'dict') await p(alice.gc.forceImmediately)() + const msgs2 = [] + for await (const msg of alice.db.msgs()) { + msgs2.push(msg) + } // Assert situation after GC assert.deepEqual( - getFields([...alice.db.msgs()]), + getFields(msgs2), [25, 'ALICE'], 'alice has only field root msgs' ) @@ -97,12 +105,12 @@ test('Dict ghosts', async (t) => { 'log stats after' ) - assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased') - assert.ok(isDeleted(alice.db.get(msgID1)), 'msg1 deleted') - assert.ok(isDeleted(alice.db.get(msgID2)), 'msg2 deleted') // ghost! - assert.ok(isErased(alice.db.get(msgID3)), 'msg3 erased') - assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists') - assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists') + assert.ok(isErased(await p(alice.db.get)(mootID)), 'moot by def erased') + assert.ok(isDeleted(await p(alice.db.get)(msgID1)), 'msg1 deleted') + assert.ok(isDeleted(await p(alice.db.get)(msgID2)), 'msg2 deleted') // ghost! + assert.ok(isErased(await p(alice.db.get)(msgID3)), 'msg3 erased') + assert.ok(isPresent(await p(alice.db.get)(msgID4)), 'msg4 exists') + assert.ok(isPresent(await p(alice.db.get)(msgID5)), 'msg5 exists') assert.deepEqual(alice.db.ghosts.get(dictID), [msgID2]) diff --git a/test/feed-decay.test.js b/test/feed-decay.test.js index 2f04c3d..2f1809c 100644 --- a/test/feed-decay.test.js +++ b/test/feed-decay.test.js @@ -26,8 +26,12 @@ test('Feed decay', async (t) => { }) } + let msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A0', 'A1', 'A2', 'A3', 'A4'], 'alice has the whole feed' ) @@ -39,8 +43,12 @@ test('Feed decay', async (t) => { await p(alice.gc.forceImmediately)() + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A2', 'A3', 'A4'], 'alice has only latest 3 msgs in the feed' ) diff --git a/test/feed-holes.test.js b/test/feed-holes.test.js index 3eba8ed..9b9bd6d 100644 --- a/test/feed-holes.test.js +++ b/test/feed-holes.test.js @@ -28,8 +28,12 @@ test('Feed holes', async (t) => { posts.push(rec.id) } + let msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9'], 'alice has the whole feed' ) @@ -40,8 +44,12 @@ test('Feed holes', async (t) => { await p(alice.db.erase)(posts[6]) // vital as trail from A7 assert('alice deleted the middle part of the feed') + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A0', 'A1', 'A2', /* */ 'A7', 'A8', 'A9'], 'alice has the beginning and the end of the feed' ) @@ -63,8 +71,12 @@ test('Feed holes', async (t) => { await p(alice.gc.forceImmediately)() assert.deepEqual(calledErase, [posts[2]], 'erased A2') + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), [/* */ 'A7', 'A8', 'A9'], 'alice has only the end of the feed' ) diff --git a/test/orphan-weave.test.js b/test/orphan-weave.test.js index 38452ff..353bd32 100644 --- a/test/orphan-weave.test.js +++ b/test/orphan-weave.test.js @@ -1,7 +1,7 @@ const test = require('node:test') const assert = require('node:assert') const p = require('node:util').promisify -const Keypair = require('ppppp-keypair') +const Keypair = require('pzp-keypair') const { createPeer } = require('./util') const bobKeypair = Keypair.generate('ed25519', 'bob') @@ -53,8 +53,12 @@ test('Orphan weave msgs', async (t) => { tangles: [threadRoot.id], }) + let msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['B0', 'A1', 'C1'], 'alice has the full thread' ) @@ -62,8 +66,12 @@ test('Orphan weave msgs', async (t) => { await p(alice.db.del)(threadRoot.id) assert('alice deleted the root') + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A1', 'C1'], 'alice has only thread replies' ) @@ -76,8 +84,12 @@ test('Orphan weave msgs', async (t) => { await p(alice.gc.forceImmediately)() + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A1'], 'alice does not have the thread, except her own reply' ) diff --git a/test/schedule.test.js b/test/schedule.test.js index 0213a5f..7777aae 100644 --- a/test/schedule.test.js +++ b/test/schedule.test.js @@ -25,8 +25,12 @@ test('Cleanup is scheduled automatically', async (t) => { }) } + let msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A0', 'A1', 'A2', 'A3', 'A4'], 'alice has the whole feed' ) @@ -39,8 +43,12 @@ test('Cleanup is scheduled automatically', async (t) => { alice.gc.start(4 * 1024) // 4kB, approximately 8 messages await p(setTimeout)(3000) + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A2', 'A3', 'A4'], 'alice has only latest 3 msgs in the feed' ) @@ -68,8 +76,12 @@ test('Compaction is scheduled automatically', async (t) => { msgIDs.push(rec.id) } + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A', 'B', 'C', 'D', 'E'], 'alice has 5 messages' ) @@ -79,8 +91,12 @@ test('Compaction is scheduled automatically', async (t) => { await p(alice.db.del)(msgIDs[3]) await p(alice.db.del)(msgIDs[4]) + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['C'], 'alice has 1 message before compaction' ) @@ -95,8 +111,12 @@ test('Compaction is scheduled automatically', async (t) => { alice.gc.start(6 * 1024) // 6kB, approximately 12 messages await p(setTimeout)(3000) + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['C'], 'alice has 1 message after compaction' ) @@ -129,8 +149,12 @@ test('start() will automatically stop()', async (t) => { }) } + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A0', 'A1', 'A2', 'A3', 'A4'], 'alice has the whole feed' ) @@ -138,8 +162,12 @@ test('start() will automatically stop()', async (t) => { alice.gc.start(4 * 1024) // 4kB, approximately 8 messages await p(setTimeout)(3000) + msgs = [] + for await (const msg of alice.db.msgs()) { + msgs.push(msg) + } assert.deepEqual( - getTexts([...alice.db.msgs()]), + getTexts(msgs), ['A2', 'A3', 'A4'], 'alice has only latest 3 msgs in the feed' ) diff --git a/test/util.js b/test/util.js index cd64f6f..946d9fd 100644 --- a/test/util.js +++ b/test/util.js @@ -1,15 +1,15 @@ const OS = require('node:os') const Path = require('node:path') const rimraf = require('rimraf') -const caps = require('ppppp-caps') -const Keypair = require('ppppp-keypair') +const caps = require('pzp-caps') +const Keypair = require('pzp-keypair') function createPeer(config) { if (config.name) { const name = config.name const tmp = OS.tmpdir() config.global ??= {} - config.global.path ??= Path.join(tmp, `ppppp-gc-${name}-${Date.now()}`) + config.global.path ??= Path.join(tmp, `pzp-gc-${name}-${Date.now()}`) config.global.keypair ??= Keypair.generate('ed25519', name) delete config.name } @@ -27,10 +27,10 @@ function createPeer(config) { return require('secret-stack/bare')() .use(require('secret-stack/plugins/net')) .use(require('secret-handshake-ext/secret-stack')) - .use(require('ppppp-db')) - .use(require('ppppp-dict')) - .use(require('ppppp-set')) - .use(require('ppppp-goals')) + .use(require('pzp-db')) + .use(require('pzp-dict')) + .use(require('pzp-set')) + .use(require('pzp-goals')) .use(require('ssb-box')) .use(require('../lib')) .call(null, {