diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml deleted file mode 100644 index b3558e2..0000000 --- a/.github/workflows/node.js.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 10 - - strategy: - matrix: - node-version: [18.x, 20.x] - - steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: npm test diff --git a/.woodpecker.yaml b/.woodpecker.yaml new file mode 100644 index 0000000..d254f8d --- /dev/null +++ b/.woodpecker.yaml @@ -0,0 +1,13 @@ +matrix: + NODE_VERSION: + - 18 + - 20 + +steps: + test: + when: + event: [push] + image: node:${NODE_VERSION} + commands: + - npm install + - npm test \ No newline at end of file diff --git a/README.md b/README.md index fe23e21..ca3f49c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -**Work in progress** +# pzp-conductor + +PZP manager that sets tangle goals ## Installation -We're not on npm yet. In your package.json, include this as - ```js -"ppppp-conductor": "github:staltz/ppppp-conductor" +npm install pzp-conductor ``` diff --git a/lib/index.js b/lib/index.js index db49fb2..13d3e3c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,26 +1,34 @@ const makeDebug = require('debug') -const MsgV4 = require('ppppp-db/msg-v4') +const MsgV4 = require('pzp-db/msg-v4') /** - * @typedef {ReturnType} PPPPPDB - * @typedef {ReturnType} PPPPPGoal - * @typedef {import('ppppp-goals').GoalDSL} GoalDSL - * @typedef {ReturnType} PPPPPSet - * @typedef {ReturnType} PPPPPDict - * @typedef {ReturnType} PPPPPSync - * @typedef {ReturnType} PPPPPGC + * @typedef {ReturnType} PZPDB + * @typedef {ReturnType} PZPGoal + * @typedef {import('pzp-goals').GoalDSL} GoalDSL + * @typedef {ReturnType} PZPSet + * @typedef {ReturnType} PZPDict + * @typedef {ReturnType} PZPSync + * @typedef {ReturnType} PZPGC * @typedef {`${string}@${GoalDSL}`} Rule * @typedef {[Array, Array]} Rules * @typedef {{ - * db: PPPPPDB, - * goals: PPPPPGoal, - * set: PPPPPSet, - * sync: PPPPPSync, - * gc: PPPPPGC, - * dict: PPPPPDict | null, + * db: PZPDB, + * goals: PZPGoal, + * set: PZPSet, + * sync: PZPSync, + * gc: PZPGC, + * dict: PZPDict | null, * }} Peer */ +/** + * @template T + * @typedef {T extends void ? + * (...args: [Error] | []) => void : + * (...args: [Error] | [null, T]) => void + * } CB + */ + /** * @param {any} rule * @returns {[string, GoalDSL]} @@ -81,7 +89,7 @@ function initConductor(peer, config) { */ const MAX_DECENT_MAXBYTES = 100 * 1024 * 1024 // 100 MB - const debug = makeDebug('ppppp:conductor') + const debug = makeDebug('pzp:conductor') /** * @param {Array} rules @@ -213,57 +221,63 @@ function initConductor(peer, config) { /** * Starts automatic sync and garbage collection. - * Assumes that PPPPP Set has been loaded with the same accountID. + * Assumes that PZP Set has been loaded with the same accountID. * * @param {string} myID * @param {[Array, Array]} rules * @param {number} maxBytes + * @param {CB} cb */ - function start(myID, rules, maxBytes) { + function start(myID, rules, maxBytes, cb) { if (maxBytes < MIN_MAXBYTES) { // prettier-ignore - throw new Error(`ppppp-conductor maxBytes must be at least ${MIN_MAXBYTES} bytes, got ${maxBytes}`) + return cb(Error(`pzp-conductor maxBytes must be at least ${MIN_MAXBYTES} bytes, got ${maxBytes}`)) } if (maxBytes > MAX_DECENT_MAXBYTES) { // prettier-ignore debug('WARNING. maxBytes is too big, we recommend at most %s bytes', MAX_DECENT_MAXBYTES) } - const follows = peer.set.values('follows') - const numFollows = follows.length - const [myRules, theirRules] = validateRules(rules, numFollows, maxBytes) + peer.set.values('follows', null, (err, follows) => { + if (err) return cb(err) - // Set up goals for my account and each account I follow - setupAccountGoals(myID, myRules) - for (const theirID of follows) { - setupAccountGoals(theirID, theirRules) - } - // @ts-ignore - peer.set.watch(({ event, subdomain, value }) => { - const theirID = value - if (subdomain === 'follows' && event === 'add') { + const numFollows = follows.length + const [myRules, theirRules] = validateRules(rules, numFollows, maxBytes) + + // Set up goals for my account and each account I follow + setupAccountGoals(myID, myRules) + for (const theirID of follows) { setupAccountGoals(theirID, theirRules) } - if (subdomain === 'follows' && event === 'del') { - teardownAccountGoals(theirID, theirRules) - } - if (subdomain === 'blocks' && event === 'add') { - teardownAccountGoals(theirID, theirRules) - } + // @ts-ignore + peer.set.watch(({ event, subdomain, value }) => { + const theirID = value + if (subdomain === 'follows' && event === 'add') { + setupAccountGoals(theirID, theirRules) + } + if (subdomain === 'follows' && event === 'del') { + teardownAccountGoals(theirID, theirRules) + } + if (subdomain === 'blocks' && event === 'add') { + teardownAccountGoals(theirID, theirRules) + } + }) + + // Figure out ghost span for each account + const totalGhostableFeeds = + countGhostableFeeds(myRules) + + numFollows * countGhostableFeeds(theirRules) + const TOTAL_GHOSTS = ESTIMATE_TOTAL_GHOST_BYTES / MSG_ID_BYTES + const ghostSpan = Math.round(TOTAL_GHOSTS / totalGhostableFeeds) + peer.set.setGhostSpan(ghostSpan) + peer.dict?.setGhostSpan(ghostSpan) + + // Kick off garbage collection and synchronization + peer.gc.start(maxBytes) + peer.sync.start() + + cb() }) - - // Figure out ghost span for each account - const totalGhostableFeeds = - countGhostableFeeds(myRules) + - numFollows * countGhostableFeeds(theirRules) - const TOTAL_GHOSTS = ESTIMATE_TOTAL_GHOST_BYTES / MSG_ID_BYTES - const ghostSpan = Math.round(TOTAL_GHOSTS / totalGhostableFeeds) - peer.set.setGhostSpan(ghostSpan) - peer.dict?.setGhostSpan(ghostSpan) - - // Kick off garbage collection and synchronization - peer.gc.start(maxBytes) - peer.sync.start() } return { diff --git a/package.json b/package.json index 8508fc3..401246a 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { - "name": "ppppp-conductor", - "version": "1.0.0", - "description": "PPPPP manager that sets tangle goals", + "name": "pzp-conductor", + "version": "0.0.1", + "description": "PZP manager that sets tangle goals", "author": "Andre Staltz ", "license": "MIT", - "homepage": "https://github.com/staltz/ppppp-conductor", + "homepage": "https://codeberg.org/pzp/pzp-conductor", "repository": { "type": "git", - "url": "git@github.com:staltz/ppppp-conductor.git" + "url": "git@codeberg.org:pzp/pzp-conductor.git" }, "type": "commonjs", "main": "index.js", @@ -26,18 +26,18 @@ "debug": "^4.3.4" }, "devDependencies": { - "@types/node": "18.x", + "@types/node": "^18.19.31", "@types/debug": "4.1.9", "bs58": "^5.0.0", "c8": "7", - "ppppp-caps": "github:staltz/ppppp-caps#93fa810b9a40b78aef4872d4c2a8412cccb52929", - "ppppp-db": "github:staltz/ppppp-db#cf1532965ea1d16929ed2291a9b737a4ce74caac", - "ppppp-dict": "github:staltz/ppppp-dict#c40d51be6cb96982b4fe691a292b3c12b6f49a36", - "ppppp-gc": "github:staltz/ppppp-gc#9075f983d8fa9a13c18a63451a78bed5912e78d0", - "ppppp-goals": "github:staltz/ppppp-goals#46a8d8889c668cf291607963fd7301f21aa634b5", - "ppppp-keypair": "github:staltz/ppppp-keypair#c33980c580e33f9a35cb0c672b916ec9fe8b4c6d", - "ppppp-set": "github:staltz/ppppp-set#07c3e295b2d09d2d6c3ac6b5b93ad2ea80698452", - "ppppp-sync": "github:staltz/ppppp-sync#93f00dbd04267f472fbf2f3ae63495092d3a921e", + "pzp-caps": "^1.0.0", + "pzp-db": "^1.0.1", + "pzp-dict": "^1.0.0", + "pzp-gc": "^1.0.0", + "pzp-goals": "^1.0.0", + "pzp-keypair": "^1.0.0", + "pzp-set": "^1.0.0", + "pzp-sync": "^1.0.0", "prettier": "^2.6.2", "pretty-quick": "^3.1.3", "rimraf": "^4.4.0", diff --git a/test/follow-feeds.test.js b/test/follow-feeds.test.js index d665b41..264120b 100644 --- a/test/follow-feeds.test.js +++ b/test/follow-feeds.test.js @@ -3,7 +3,11 @@ const assert = require('node:assert') const p = require('node:util').promisify const { createPeer } = require('./util') -function getTexts(msgs) { +async function getTexts(iter) { + const msgs = [] + for await (i of iter) { + msgs.push(i) + } return msgs.filter((msg) => msg.data?.text).map((msg) => msg.data.text) } @@ -18,7 +22,7 @@ test('Sets goals according to input rules', async (t) => { }) await p(alice.set.load)(aliceID) - alice.conductor.start( + await p(alice.conductor.start)( aliceID, [['posts@newest-100', 'hubs@set', 'profile@dict']], 64_000_000 @@ -107,15 +111,15 @@ test('Replicate selected feeds of followed accounts', async (t) => { // Alice follows Bob, but not Carol assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - alice.conductor.start(aliceID, [['post@all'], ['post@all']], 64_000_000) - bob.conductor.start(bobID, [['post@all'], ['post@all']], 64_000_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 64_000_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 64_000_000) const aliceDialingBob = await p(alice.connect)(bob.getAddress()) const aliceDialingCarol = await p(alice.connect)(carol.getAddress()) await p(setTimeout)(1000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4', /* */ 'B0', 'B1', 'B2', 'B3', 'B4'], 'alice has alice and bob posts' ) @@ -185,15 +189,15 @@ test('GC selected feeds of followed accounts', async (t) => { // Alice follows Bob, but not Carol assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - alice.conductor.start(aliceID, [['post@all'], ['post@all']], 64_000_000) - bob.conductor.start(bobID, [['post@all'], ['post@all']], 64_000_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 64_000_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 64_000_000) const aliceDialingBob = await p(alice.connect)(bob.getAddress()) const aliceDialingCarol = await p(alice.connect)(carol.getAddress()) await p(setTimeout)(1000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4', /* */ 'B0', 'B1', 'B2', 'B3', 'B4'], 'alice has alice and bob posts' ) @@ -201,13 +205,13 @@ test('GC selected feeds of followed accounts', async (t) => { await p(aliceDialingBob.close)(true) await p(aliceDialingCarol.close)(true) - alice.conductor.start(aliceID, [['post@all'], ['post@newest-2']], 8_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@newest-2']], 8_000) const aliceDialingBob2 = await p(alice.connect)(bob.getAddress()) const aliceDialingCarol2 = await p(alice.connect)(carol.getAddress()) await p(setTimeout)(1000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4', /* */ 'B3', 'B4'], 'alice has alice and bob posts' ) @@ -277,15 +281,15 @@ test('GC recently-unfollowed accounts', async (t) => { // Alice follows Bob, but not Carol assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - alice.conductor.start(aliceID, [['post@all'], ['post@all']], 4_000) - bob.conductor.start(bobID, [['post@all'], ['post@all']], 4_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 4_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 4_000) const aliceDialingBob = await p(alice.connect)(bob.getAddress()) const aliceDialingCarol = await p(alice.connect)(carol.getAddress()) await p(setTimeout)(2000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4', /* */ 'B0', 'B1', 'B2', 'B3', 'B4'], 'alice has alice and bob posts' ) @@ -294,7 +298,7 @@ test('GC recently-unfollowed accounts', async (t) => { await p(setTimeout)(1000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4'], 'alice has alice posts' ) @@ -364,15 +368,15 @@ test('GC recently-blocked accounts', async (t) => { // Alice follows Bob, but not Carol assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - alice.conductor.start(aliceID, [['post@all'], ['post@all']], 4_000) - bob.conductor.start(bobID, [['post@all'], ['post@all']], 4_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 4_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 4_000) const aliceDialingBob = await p(alice.connect)(bob.getAddress()) const aliceDialingCarol = await p(alice.connect)(carol.getAddress()) await p(setTimeout)(2000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4', /* */ 'B0', 'B1', 'B2', 'B3', 'B4'], 'alice has alice and bob posts' ) @@ -381,7 +385,7 @@ test('GC recently-blocked accounts', async (t) => { await p(setTimeout)(1000) assert.deepEqual( - getTexts([...alice.db.msgs()]), + await getTexts(alice.db.msgs()), ['A0', 'A1', 'A2', 'A3', 'A4'], 'alice has alice posts' ) @@ -427,8 +431,8 @@ test('Set and Dict ghost spans', async (t) => { // Alice follows Bob, but not Carol assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - alice.conductor.start(aliceID, [['post@all'], ['post@all']], 4_000) - bob.conductor.start(bobID, [['post@all'], ['post@all']], 4_000) + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 4_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 4_000) assert.equal(alice.set.getGhostSpan(), 5958, 'alice set ghost span is 2') assert.equal(alice.dict.getGhostSpan(), 5958, 'alice set ghost span is 2') diff --git a/test/util.js b/test/util.js index cd7da2d..602bcc2 100644 --- a/test/util.js +++ b/test/util.js @@ -1,15 +1,17 @@ const OS = require('node:os') const Path = require('node:path') const rimraf = require('rimraf') -const caps = require('ppppp-caps') -const Keypair = require('ppppp-keypair') +/** @type {string} */ +// @ts-ignore +const caps = require('pzp-caps') +const Keypair = require('pzp-keypair') function createPeer(config) { if (config.name) { const name = config.name const tmp = OS.tmpdir() config.global ??= {} - config.global.path ??= Path.join(tmp, `ppppp-conduct-${name}-${Date.now()}`) + config.global.path ??= Path.join(tmp, `pzp-conduct-${name}-${Date.now()}`) config.global.keypair ??= Keypair.generate('ed25519', name) delete config.name } @@ -24,16 +26,19 @@ function createPeer(config) { } rimraf.sync(config.global.path) + // @ts-ignore return require('secret-stack/bare')() + // @ts-ignore .use(require('secret-stack/plugins/net')) .use(require('secret-handshake-ext/secret-stack')) + // @ts-ignore .use(require('ssb-box')) - .use(require('ppppp-db')) - .use(require('ppppp-set')) - .use(require('ppppp-dict')) - .use(require('ppppp-goals')) - .use(require('ppppp-sync')) - .use(require('ppppp-gc')) + .use(require('pzp-db')) + .use(require('pzp-set')) + .use(require('pzp-dict')) + .use(require('pzp-goals')) + .use(require('pzp-sync')) + .use(require('pzp-gc')) .use(require('../lib')) .call(null, { shse: { caps }, diff --git a/tsconfig.json b/tsconfig.json index bd2acd5..02ae111 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,13 +1,22 @@ { - "include": ["lib/**/*.js"], - "exclude": ["coverage/", "node_modules/", "test/"], + "include": [ + "lib/**/*.js" + ], + "exclude": [ + "coverage/", + "node_modules/", + "test/" + ], "compilerOptions": { "checkJs": true, "declaration": true, "emitDeclarationOnly": true, "exactOptionalPropertyTypes": true, "forceConsistentCasingInFileNames": true, - "lib": ["es2022", "dom"], + "lib": [ + "es2022", + "dom" + ], "module": "node16", "skipLibCheck": true, "strict": true,