pzp
/
pzp-gc
mirror of https://codeberg.org/pzp/pzp-gc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename to pzp and upgrade modules

This commit is contained in:
Jacob Karlsson 2024-04-30 17:17:26 +02:00
parent 9075f983d8
commit b0157ed984
11 changed files with 213 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/node.js.yml vendored
View File

@ -1,25 +0,0 @@
name: CI
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
matrix:
node-version: [18.x, 20.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm test

13
.woodpecker.yaml Normal file
View File

@ -0,0 +1,13 @@
matrix:
NODE_VERSION:
- 18
- 20
steps:
test:
when:
event: [push]
image: node:${NODE_VERSION}
commands:
- npm install
- npm test

6
README.md
View File

@ -1,9 +1,7 @@
**Work in progress**
# pzp-gc
## Installation
We're not on npm yet. In your package.json, include this as
```js
"ppppp-gc": "github:staltz/ppppp-gc"
npm install pzp-gc
```

37
lib/index.js
View File

@ -2,9 +2,20 @@
const multicb = require('multicb')
const makeDebug = require('debug')
// @ts-ignore
const p = (fn) => (...args) => {
return new Promise((res, rej) => {
// @ts-ignore
fn(...args, (err, val) => {
if (err) return rej(err)
return res(val)
})
})
}
/**
* @typedef {ReturnType<import('ppppp-db').init>} PPPPPDB
* @typedef {ReturnType<import('ppppp-goals').init>} PPPPPGoal
* @typedef {ReturnType<import('pzp-db').init>} pzpDB
* @typedef {ReturnType<import('pzp-goals').init>} pzpGoal
* @typedef {{
* gc: {
* maxLogBytes: number
@ -23,12 +34,12 @@ const makeDebug = require('debug')
*/
/**
* @param {{ db: PPPPPDB, goals: PPPPPGoal }} peer
* @param {{ db: pzpDB, goals: pzpGoal }} peer
* @param {Config} config
*/
function initGC(peer, config) {
// State
const debug = makeDebug('ppppp:gc')
const debug = makeDebug('pzp:gc')
let stopMonitoringLogSize = /** @type {CallableFunction | null} */ (null)
let hasCleanupScheduled = false
@ -47,9 +58,10 @@ function initGC(peer, config) {
/**
* Deletes messages that don't correspond to any goal.
* @private
* @param {CB<void>} cb
* @return {Promise<void>}
*/
function cleanup(cb) {
async function cleanup() {
return new Promise(async (res, rej) => {
debug('Cleanup started')
const startTime = Date.now()
const done = multicb({ pluck: 1 })
@ -66,10 +78,10 @@ function initGC(peer, config) {
}
let waiting = false
for (const rec of peer.db.records()) {
for await (const rec of peer.db.records()) {
if (!rec.msg) continue
const { id: msgID, msg } = rec
const [purpose, details] = peer.goals.getMsgPurpose(msgID, msg)
const [purpose, details] = await p(peer.goals.getMsgPurpose)(msgID, msg)
switch (purpose) {
case 'goal': {
continue // don't cleanup
@ -102,7 +114,7 @@ function initGC(peer, config) {
continue
}
default: {
cb(new Error('Unreachable'))
rej(new Error('Unreachable'))
return
}
}
@ -116,8 +128,9 @@ function initGC(peer, config) {
const duration = Date.now() - startTime
if (err) debug('Cleanup ended with an error %s', err.message ?? err)
else debug('Cleanup completed in %sms', duration)
cb()
res()
}
})
}
/**
@ -203,7 +216,7 @@ function initGC(peer, config) {
if (needsCompaction) reportCompactionNeed(percentDeleted, stats)
hasCleanupScheduled = true
if (needsCleanup) {
cleanup(() => {
cleanup().finally(() => {
compact(() => {
hasCleanupScheduled = false
})
@ -253,7 +266,7 @@ function initGC(peer, config) {
*/
function forceImmediately(cb) {
debug('Force clean and compact immediately')
cleanup(() => {
cleanup().finally(() => {
compact(cb)
})
}

22
package.json
View File

@ -1,13 +1,13 @@
{
"name": "ppppp-gc",
"version": "1.0.0",
"description": "PPPPP garbage collector",
"name": "pzp-gc",
"version": "0.0.1",
"description": "PZP garbage collector",
"author": "Andre Staltz <contact@staltz.com>",
"license": "MIT",
"homepage": "https://github.com/staltz/ppppp-gc",
"homepage": "https://codeberg.org/pzp/pzp-gc",
"repository": {
"type": "git",
"url": "git@github.com:staltz/ppppp-gc.git"
"url": "git@codeberg.org:pzp/pzp-gc.git"
},
"main": "index.js",
"files": [
@ -31,12 +31,12 @@
"@types/debug": "4.1.9",
"bs58": "^5.0.0",
"c8": "7",
"ppppp-caps": "github:staltz/ppppp-caps#93fa810b9a40b78aef4872d4c2a8412cccb52929",
"ppppp-db": "github:staltz/ppppp-db#667b33779d98aff12a9b0cd2d7c80469a95cd04e",
"ppppp-dict": "github:staltz/ppppp-dict#6f0ff4e3383a8c18b766949f6db9b51460ecb640",
"ppppp-goals": "github:staltz/ppppp-goals#f862c2de624649906a4375711f3813db3b94a2ca",
"ppppp-keypair": "github:staltz/ppppp-keypair#61ef4420578f450dc2cc7b1efc1c5a691a871c74",
"ppppp-set": "github:staltz/ppppp-set#8983ba29f03db95a76b4bd9a55aa4392b350fdbb",
"pzp-caps": "^1.0.0",
"pzp-db": "^1.0.1",
"pzp-dict": "^1.0.0",
"pzp-goals": "^1.0.0",
"pzp-keypair": "^1.0.0",
"pzp-set": "^1.0.0",
"prettier": "^2.6.2",
"pretty-quick": "^3.1.3",
"rimraf": "^4.4.0",

38
test/dict-ghosts.test.js
View File

@ -51,7 +51,7 @@ test('Dict ghosts', async (t) => {
let msgID3
let msgID4
let msgID5
for (const rec of alice.db.records()) {
for await (const rec of alice.db.records()) {
if (rec.msg.metadata.dataSize === 0) mootID = rec.id
if (rec.msg.data?.update?.name === 'alice') msgID1 = rec.id
if (rec.msg.data?.update?.age === 24) msgID2 = rec.id
@ -60,18 +60,22 @@ test('Dict ghosts', async (t) => {
if (rec.msg.data?.update?.name === 'ALICE') msgID5 = rec.id
}
const msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
// Assert situation before GC
assert.deepEqual(
getFields([...alice.db.msgs()]),
getFields(msgs),
['alice', 24, 'Alice', 25, 'ALICE'],
'has all dict msgs'
)
assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased')
assert.ok(isPresent(alice.db.get(msgID1)), 'msg1 exists')
assert.ok(isPresent(alice.db.get(msgID2)), 'msg2 exists')
assert.ok(isPresent(alice.db.get(msgID3)), 'msg3 exists')
assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists')
assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists')
assert.ok(isErased(await p(alice.db.get)(mootID)), 'moot by def erased')
assert.ok(isPresent(await p(alice.db.get)(msgID1)), 'msg1 exists')
assert.ok(isPresent(await p(alice.db.get)(msgID2)), 'msg2 exists')
assert.ok(isPresent(await p(alice.db.get)(msgID3)), 'msg3 exists')
assert.ok(isPresent(await p(alice.db.get)(msgID4)), 'msg4 exists')
assert.ok(isPresent(await p(alice.db.get)(msgID5)), 'msg5 exists')
assert.deepEqual(
await p(alice.db.log.stats)(),
@ -84,9 +88,13 @@ test('Dict ghosts', async (t) => {
alice.goals.set(dictID, 'dict')
await p(alice.gc.forceImmediately)()
const msgs2 = []
for await (const msg of alice.db.msgs()) {
msgs2.push(msg)
}
// Assert situation after GC
assert.deepEqual(
getFields([...alice.db.msgs()]),
getFields(msgs2),
[25, 'ALICE'],
'alice has only field root msgs'
)
@ -97,12 +105,12 @@ test('Dict ghosts', async (t) => {
'log stats after'
)
assert.ok(isErased(alice.db.get(mootID)), 'moot by def erased')
assert.ok(isDeleted(alice.db.get(msgID1)), 'msg1 deleted')
assert.ok(isDeleted(alice.db.get(msgID2)), 'msg2 deleted') // ghost!
assert.ok(isErased(alice.db.get(msgID3)), 'msg3 erased')
assert.ok(isPresent(alice.db.get(msgID4)), 'msg4 exists')
assert.ok(isPresent(alice.db.get(msgID5)), 'msg5 exists')
assert.ok(isErased(await p(alice.db.get)(mootID)), 'moot by def erased')
assert.ok(isDeleted(await p(alice.db.get)(msgID1)), 'msg1 deleted')
assert.ok(isDeleted(await p(alice.db.get)(msgID2)), 'msg2 deleted') // ghost!
assert.ok(isErased(await p(alice.db.get)(msgID3)), 'msg3 erased')
assert.ok(isPresent(await p(alice.db.get)(msgID4)), 'msg4 exists')
assert.ok(isPresent(await p(alice.db.get)(msgID5)), 'msg5 exists')
assert.deepEqual(alice.db.ghosts.get(dictID), [msgID2])

12
test/feed-decay.test.js
View File

@ -26,8 +26,12 @@ test('Feed decay', async (t) => {
})
}
let msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A0', 'A1', 'A2', 'A3', 'A4'],
'alice has the whole feed'
)
@ -39,8 +43,12 @@ test('Feed decay', async (t) => {
await p(alice.gc.forceImmediately)()
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A2', 'A3', 'A4'],
'alice has only latest 3 msgs in the feed'
)

18
test/feed-holes.test.js
View File

@ -28,8 +28,12 @@ test('Feed holes', async (t) => {
posts.push(rec.id)
}
let msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9'],
'alice has the whole feed'
)
@ -40,8 +44,12 @@ test('Feed holes', async (t) => {
await p(alice.db.erase)(posts[6]) // vital as trail from A7
assert('alice deleted the middle part of the feed')
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A0', 'A1', 'A2', /* */ 'A7', 'A8', 'A9'],
'alice has the beginning and the end of the feed'
)
@ -63,8 +71,12 @@ test('Feed holes', async (t) => {
await p(alice.gc.forceImmediately)()
assert.deepEqual(calledErase, [posts[2]], 'erased A2')
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
[/* */ 'A7', 'A8', 'A9'],
'alice has only the end of the feed'
)

20
test/orphan-weave.test.js
View File

@ -1,7 +1,7 @@
const test = require('node:test')
const assert = require('node:assert')
const p = require('node:util').promisify
const Keypair = require('ppppp-keypair')
const Keypair = require('pzp-keypair')
const { createPeer } = require('./util')
const bobKeypair = Keypair.generate('ed25519', 'bob')
@ -53,8 +53,12 @@ test('Orphan weave msgs', async (t) => {
tangles: [threadRoot.id],
})
let msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['B0', 'A1', 'C1'],
'alice has the full thread'
)
@ -62,8 +66,12 @@ test('Orphan weave msgs', async (t) => {
await p(alice.db.del)(threadRoot.id)
assert('alice deleted the root')
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A1', 'C1'],
'alice has only thread replies'
)
@ -76,8 +84,12 @@ test('Orphan weave msgs', async (t) => {
await p(alice.gc.forceImmediately)()
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A1'],
'alice does not have the thread, except her own reply'
)

42
test/schedule.test.js
View File

@ -25,8 +25,12 @@ test('Cleanup is scheduled automatically', async (t) => {
})
}
let msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A0', 'A1', 'A2', 'A3', 'A4'],
'alice has the whole feed'
)
@ -39,8 +43,12 @@ test('Cleanup is scheduled automatically', async (t) => {
alice.gc.start(4 * 1024) // 4kB, approximately 8 messages
await p(setTimeout)(3000)
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A2', 'A3', 'A4'],
'alice has only latest 3 msgs in the feed'
)
@ -68,8 +76,12 @@ test('Compaction is scheduled automatically', async (t) => {
msgIDs.push(rec.id)
}
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A', 'B', 'C', 'D', 'E'],
'alice has 5 messages'
)
@ -79,8 +91,12 @@ test('Compaction is scheduled automatically', async (t) => {
await p(alice.db.del)(msgIDs[3])
await p(alice.db.del)(msgIDs[4])
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['C'],
'alice has 1 message before compaction'
)
@ -95,8 +111,12 @@ test('Compaction is scheduled automatically', async (t) => {
alice.gc.start(6 * 1024) // 6kB, approximately 12 messages
await p(setTimeout)(3000)
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['C'],
'alice has 1 message after compaction'
)
@ -129,8 +149,12 @@ test('start() will automatically stop()', async (t) => {
})
}
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A0', 'A1', 'A2', 'A3', 'A4'],
'alice has the whole feed'
)
@ -138,8 +162,12 @@ test('start() will automatically stop()', async (t) => {
alice.gc.start(4 * 1024) // 4kB, approximately 8 messages
await p(setTimeout)(3000)
msgs = []
for await (const msg of alice.db.msgs()) {
msgs.push(msg)
}
assert.deepEqual(
getTexts([...alice.db.msgs()]),
getTexts(msgs),
['A2', 'A3', 'A4'],
'alice has only latest 3 msgs in the feed'
)

14
test/util.js
View File

@ -1,15 +1,15 @@
const OS = require('node:os')
const Path = require('node:path')
const rimraf = require('rimraf')
const caps = require('ppppp-caps')
const Keypair = require('ppppp-keypair')
const caps = require('pzp-caps')
const Keypair = require('pzp-keypair')
function createPeer(config) {
if (config.name) {
const name = config.name
const tmp = OS.tmpdir()
config.global ??= {}
config.global.path ??= Path.join(tmp, `ppppp-gc-${name}-${Date.now()}`)
config.global.path ??= Path.join(tmp, `pzp-gc-${name}-${Date.now()}`)
config.global.keypair ??= Keypair.generate('ed25519', name)
delete config.name
}
@ -27,10 +27,10 @@ function createPeer(config) {
return require('secret-stack/bare')()
.use(require('secret-stack/plugins/net'))
.use(require('secret-handshake-ext/secret-stack'))
.use(require('ppppp-db'))
.use(require('ppppp-dict'))
.use(require('ppppp-set'))
.use(require('ppppp-goals'))
.use(require('pzp-db'))
.use(require('pzp-dict'))
.use(require('pzp-set'))
.use(require('pzp-goals'))
.use(require('ssb-box'))
.use(require('../lib'))
.call(null, {