diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml deleted file mode 100644 index b3558e2..0000000 --- a/.github/workflows/node.js.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 10 - - strategy: - matrix: - node-version: [18.x, 20.x] - - steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: npm test diff --git a/.woodpecker.yaml b/.woodpecker.yaml new file mode 100644 index 0000000..d254f8d --- /dev/null +++ b/.woodpecker.yaml @@ -0,0 +1,13 @@ +matrix: + NODE_VERSION: + - 18 + - 20 + +steps: + test: + when: + event: [push] + image: node:${NODE_VERSION} + commands: + - npm install + - npm test \ No newline at end of file diff --git a/README.md b/README.md index 6b8a7dd..90f3d41 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -**Work in progress** +# pzp-sync + +PZP replication using Kleppmann's hash graph sync ## Installation -We're not on npm yet. In your package.json, include this as - -```js -"ppppp-sync": "github:staltz/ppppp-sync" +``` +npm install pzp-sync ``` diff --git a/lib/algorithm.js b/lib/algorithm.js index 01fc722..93c29a5 100644 --- a/lib/algorithm.js +++ b/lib/algorithm.js @@ -1,16 +1,16 @@ const p = require('promisify-4loc') const { BloomFilter } = require('bloom-filters') -const MsgV4 = require('ppppp-db/msg-v4') +const MsgV4 = require('pzp-db/msg-v4') const makeDebug = require('debug') -const debug = makeDebug('ppppp:sync') +const debug = makeDebug('pzp:sync') const { EMPTY_RANGE, isEmptyRange, estimateMsgCount } = require('./range') /** - * @typedef {ReturnType} PPPPPDB - * @typedef {ReturnType} PPPPPDict - * @typedef {ReturnType} PPPPPSet - * @typedef {import('ppppp-db/msg-v4').Msg} Msg - * @typedef {import('ppppp-goals').Goal} Goal + * @typedef {ReturnType} PZPDB + * @typedef {ReturnType} PZPDict + * @typedef {ReturnType} PZPSet + * @typedef {import('pzp-db/msg-v4').Msg} Msg + * @typedef {import('pzp-goals').Goal} Goal * @typedef {import('./range').Range} Range * @typedef {string} MsgID */ @@ -28,7 +28,7 @@ class Algorithm { /** @type {ConstructorParameters[0]} */ #peer - /** @param {{ db: PPPPPDB, dict: PPPPPDict, set: PPPPPSet }} peer */ + /** @param {{ db: PZPDB, dict: PZPDict, set: PZPSet }} peer */ constructor(peer) { this.#peer = peer } @@ -38,14 +38,14 @@ class Algorithm { * the given tangle known by the `rootID`. * * @param {string} rootID - * @returns {Range} + * @returns {Promise} */ - haveRange(rootID) { - const rootMsg = this.#peer.db.get(rootID) + async haveRange(rootID) { + const rootMsg = await p(this.#peer.db.get)(rootID) if (!rootMsg) return EMPTY_RANGE let minDepth = Number.MAX_SAFE_INTEGER let maxDepth = 0 - for (const rec of this.#peer.db.records()) { + for await (const rec of this.#peer.db.records()) { if (!rec?.msg?.data && rec.id !== rootID) continue const tangles = rec.msg.metadata.tangles if (rec.id === rootID) { @@ -118,9 +118,9 @@ class Algorithm { * @param {Range} localHave * @param {Range} remoteHave * @param {Goal?} goal - * @returns {Range} + * @returns {Promise} */ - wantRange(localHave, remoteHave, goal) { + async wantRange(localHave, remoteHave, goal) { if (!goal) return EMPTY_RANGE if (isEmptyRange(remoteHave)) return EMPTY_RANGE @@ -129,11 +129,11 @@ class Algorithm { return this.#wantAllRange(localHave, remoteHave) case 'dict': - const minDictGhostDepth = this.#peer.dict.minGhostDepth(goal.id) + const minDictGhostDepth = await p(this.#peer.dict.minGhostDepth)(goal.id) return this.#wantDictOrSetRange(minDictGhostDepth, remoteHave) case 'set': - const minSetGhostDepth = this.#peer.set.minGhostDepth(goal.id) + const minSetGhostDepth = await p(this.#peer.set.minGhostDepth)(goal.id) return this.#wantDictOrSetRange(minSetGhostDepth, remoteHave) case 'newest': @@ -159,15 +159,15 @@ class Algorithm { * @param {number} round * @param {Range} range * @param {Iterable} extraIds - * @returns {JSON} + * @returns {Promise} */ - bloomFor(rootID, round, range, extraIds = []) { + async bloomFor(rootID, round, range, extraIds = []) { const filterSize = (isEmptyRange(range) ? 2 : estimateMsgCount(range)) + countIter(extraIds) const filter = BloomFilter.create(2 * filterSize, 0.00001) if (!isEmptyRange(range)) { - const rangeMsgs = this.getMsgsInRange(rootID, range) - const accountMsgs = this.getAccountMsgsFor(rangeMsgs) + const rangeMsgs = await this.getMsgsInRange(rootID, range) + const accountMsgs = await this.getAccountMsgsFor(rangeMsgs) for (const msg of accountMsgs.concat(rangeMsgs)) { filter.add('' + round + MsgV4.getMsgID(msg)) } @@ -196,14 +196,14 @@ class Algorithm { * @param {number} round * @param {Range} range * @param {JSON} remoteBloomJSON - * @returns {Array} + * @returns {Promise>} */ - getMsgsMissing(rootID, round, range, remoteBloomJSON) { + async getMsgsMissing(rootID, round, range, remoteBloomJSON) { if (isEmptyRange(range)) return [] const remoteFilter = BloomFilter.fromJSON(remoteBloomJSON) const missing = [] - const rangeMsgs = this.getMsgsInRange(rootID, range) - const accountMsgs = this.getAccountMsgsFor(rangeMsgs) + const rangeMsgs = await this.getMsgsInRange(rootID, range) + const accountMsgs = await this.getAccountMsgsFor(rangeMsgs) for (const msg of accountMsgs.concat(rangeMsgs)) { const msgID = MsgV4.getMsgID(msg) if (!remoteFilter.has('' + round + msgID)) { @@ -218,9 +218,9 @@ class Algorithm { * `msgs`. * * @param {Array} msgs - * @returns {Array} + * @returns {Promise>} */ - getAccountMsgsFor(msgs) { + async getAccountMsgsFor(msgs) { const accountTips = /** @type {Map>} */ (new Map()) for (const msg of msgs) { if (MsgV4.isFeedMsg(msg)) { @@ -234,9 +234,9 @@ class Algorithm { const accountMsgs = [] for (const [accountID, tips] of accountTips) { - const accountTangle = this.#peer.db.getTangle(accountID) + const accountTangle = await p(this.#peer.db.getTangle)(accountID) if (!accountTangle) continue - accountMsgs.push(...accountTangle.slice([], [...tips])) + accountMsgs.push(...(await accountTangle.slice([], [...tips]))) } return accountMsgs } @@ -246,12 +246,12 @@ class Algorithm { * as msgs. * * @param {Iterable} msgIDs - * @returns {Array} + * @returns {Promise>} */ - filterAndFetchAccountMsgs(msgIDs) { + async filterAndFetchAccountMsgs(msgIDs) { const accountMsgs = [] for (const msgID of msgIDs) { - const msg = this.#peer.db.get(msgID) + const msg = await p(this.#peer.db.get)(msgID) if (msg?.metadata.account === 'self') { accountMsgs.push(msg) } @@ -265,19 +265,19 @@ class Algorithm { * * @param {string} rootID * @param {Range} range - * @returns {Array} + * @returns {Promise>} */ - getMsgsInRange(rootID, range) { + async getMsgsInRange(rootID, range) { const [minDepth, maxDepth] = range - const rootMsg = this.#peer.db.get(rootID) + const rootMsg = await p(this.#peer.db.get)(rootID) if (!rootMsg) return [] const msgs = [] if (minDepth === 0) { msgs.push(rootMsg) } - const tangle = this.#peer.db.getTangle(rootID) + const tangle = await p(this.#peer.db.getTangle)(rootID) if (!tangle) return msgs - for (const msg of tangle.slice()) { + for (const msg of await tangle.slice()) { const depth = msg.metadata.tangles[rootID]?.depth ?? 0 if (depth >= minDepth && depth <= maxDepth) { msgs.push(msg) @@ -293,15 +293,15 @@ class Algorithm { * * @param {string} rootID * @param {Set | Array} msgs - * @returns {Array} + * @returns {Promise>} */ - getTangleMsgs(rootID, msgs) { + async getTangleMsgs(rootID, msgs) { if (Array.isArray(msgs) && msgs.length === 0) return [] if (!Array.isArray(msgs) && msgs.size === 0) return [] const msgIDs = [...msgs].map((m) => typeof m === 'string' ? m : MsgV4.getMsgID(m) ) - const tangle = this.#peer.db.getTangle(rootID) + const tangle = await p(this.#peer.db.getTangle)(rootID) if (!tangle) return [] return tangle.slice(msgIDs, []) } @@ -314,7 +314,7 @@ class Algorithm { * @param {number} count */ async pruneNewest(rootID, count) { - const tangle = this.#peer.db.getTangle(rootID) + const tangle = await p(this.#peer.db.getTangle)(rootID) if (!tangle) return const sorted = tangle.topoSort() if (sorted.length <= count) return diff --git a/lib/index.js b/lib/index.js index bdf842a..0268fd5 100644 --- a/lib/index.js +++ b/lib/index.js @@ -7,10 +7,10 @@ const Algorithm = require('./algorithm') const SyncStream = require('./stream') /** - * @typedef {ReturnType} PPPPPDB - * @typedef {ReturnType} PPPPPDict - * @typedef {ReturnType} PPPPPSet - * @typedef {ReturnType} PPPPPGoals + * @typedef {ReturnType} PZPDB + * @typedef {ReturnType} PZPDict + * @typedef {ReturnType} PZPSet + * @typedef {ReturnType} PZPGoals * @typedef {import('node:events').EventEmitter} Emitter * @typedef {(cb: (err: Error) => void) => import('pull-stream').Duplex} GetDuplex * @typedef {{ pubkey: string }} SHSE @@ -29,16 +29,16 @@ function isMuxrpcMissingError(err, namespace, methodName) { /** * @param {Emitter & { - * db: PPPPPDB, - * dict: PPPPPDict, - * set: PPPPPSet, - * goals: PPPPPGoals, + * db: PZPDB, + * dict: PZPDict, + * set: PZPSet, + * goals: PZPGoals, * shse: SHSE * }} peer * @param {unknown} config */ function initSync(peer, config) { - const debug = makeDebug(`ppppp:sync`) + const debug = makeDebug(`pzp:sync`) const algo = new Algorithm(peer) let started = false diff --git a/lib/stream.js b/lib/stream.js index 1ba1b07..04de876 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -1,16 +1,17 @@ // @ts-ignore const Pipeable = require('push-stream/pipeable') +const p = require('promisify-4loc') const { isRange, isEmptyRange } = require('./range') const { isMsgId, isBloom, isMsgIds, isMsgs } = require('./util') /** - * @typedef {ReturnType} PPPPPGoals - * @typedef {ReturnType} PPPPPDB - * @typedef {import('ppppp-db').RecPresent} Rec - * @typedef {import('ppppp-db/msg-v4').Msg} Msg + * @typedef {ReturnType} PZPGoals + * @typedef {ReturnType} PZPDB + * @typedef {import('pzp-db').RecPresent} Rec + * @typedef {import('pzp-db/msg-v4').Msg} Msg * @typedef {import('./range').Range} Range * @typedef {import('./algorithm')} Algorithm - * @typedef {import('ppppp-goals').Goal} Goal + * @typedef {import('pzp-goals').Goal} Goal * @typedef {string} MsgID * @typedef {{id: string}} WithId * @typedef {WithId & {phase: 0, payload?: undefined}} Data0 @@ -23,6 +24,14 @@ const { isMsgId, isBloom, isMsgIds, isMsgs } = require('./util') * @typedef {Data0 | Data1 | Data2 | Data3 | Data4567 | Data8 | Data9} Data */ +/** + * @template T + * @typedef {[T] extends [void] ? + * (...args: [Error] | []) => void : + * (...args: [Error] | [null, T]) => void + * } CB + */ + class SyncStream extends Pipeable { #myId /** @type {Algorithm} */ @@ -31,9 +40,9 @@ class SyncStream extends Pipeable { #debug /** @type {Set} Set of tangleId */ #requested - /** @type {PPPPPDB} */ + /** @type {PZPDB} */ #db - /** @type {PPPPPGoals} */ + /** @type {PZPGoals} */ #goals /** * tangleId => have-range by local peer @@ -71,8 +80,8 @@ class SyncStream extends Pipeable { /** * @param {string} localId * @param {CallableFunction} debug - * @param {PPPPPDB} db - * @param {PPPPPGoals} goals + * @param {PZPDB} db + * @param {PZPGoals} goals * @param {Algorithm} algo */ constructor(localId, debug, db, goals, algo) { @@ -159,8 +168,8 @@ class SyncStream extends Pipeable { /** * @param {string} id */ - #sendLocalHave(id) { - const localHaveRange = this.#algo.haveRange(id) + async #sendLocalHave(id) { + const localHaveRange = await this.#algo.haveRange(id) this.#localHave.set(id, localHaveRange) this.sink.write({ id, phase: 1, payload: localHaveRange }) // prettier-ignore @@ -171,13 +180,13 @@ class SyncStream extends Pipeable { * @param {string} id * @param {Range} remoteHaveRange */ - #sendLocalHaveAndWant(id, remoteHaveRange) { + async #sendLocalHaveAndWant(id, remoteHaveRange) { // prettier-ignore this.#debug('%s Stream IN1: got remote have-range %o for %s', this.#myId, remoteHaveRange, id) this.#remoteHave.set(id, remoteHaveRange) const goal = this.#goals.get(id) - const haveRange = this.#algo.haveRange(id) - const wantRange = this.#algo.wantRange(haveRange, remoteHaveRange, goal) + const haveRange = await this.#algo.haveRange(id) + const wantRange = await this.#algo.wantRange(haveRange, remoteHaveRange, goal) this.#localHave.set(id, haveRange) this.#localWant.set(id, wantRange) this.sink.write({ id, phase: 2, payload: { haveRange, wantRange } }) @@ -190,7 +199,7 @@ class SyncStream extends Pipeable { * @param {Range} remoteHaveRange * @param {Range} remoteWantRange */ - #sendLocalWantAndInitBloom(id, remoteHaveRange, remoteWantRange) { + async #sendLocalWantAndInitBloom(id, remoteHaveRange, remoteWantRange) { // prettier-ignore this.#debug('%s Stream IN2: got remote have-range %o and want-range %o for %s', this.#myId, remoteHaveRange, remoteWantRange, id) this.#remoteHave.set(id, remoteHaveRange) @@ -198,9 +207,9 @@ class SyncStream extends Pipeable { const goal = this.#goals.get(id) const haveRange = this.#localHave.get(id) if (!haveRange) throw new Error(`Local have-range not set for ${id}`) - const localWant = this.#algo.wantRange(haveRange, remoteHaveRange, goal) + const localWant = await this.#algo.wantRange(haveRange, remoteHaveRange, goal) this.#localWant.set(id, localWant) - const localBloom0 = this.#algo.bloomFor(id, 0, localWant) + const localBloom0 = await this.#algo.bloomFor(id, 0, localWant) this.sink.write({ id, phase: 3, @@ -215,11 +224,11 @@ class SyncStream extends Pipeable { * @param {Range} remoteWantRange * @param {JSON} remoteBloom */ - #sendInitBloomRes(id, remoteWantRange, remoteBloom) { + async #sendInitBloomRes(id, remoteWantRange, remoteBloom) { // prettier-ignore this.#debug('%s Stream IN3: got remote want-range %o and bloom round 0 for %s', this.#myId, remoteWantRange, id) this.#remoteWant.set(id, remoteWantRange) - const msgIDsForThem = this.#algo.getMsgsMissing( + const msgIDsForThem = await this.#algo.getMsgsMissing( id, 0, remoteWantRange, @@ -228,7 +237,7 @@ class SyncStream extends Pipeable { this.#updateSendableMsgs(id, msgIDsForThem) const localWantRange = this.#localWant.get(id) if (!localWantRange) throw new Error(`Local want-range not set for ${id}`) - const localBloom = this.#algo.bloomFor(id, 0, localWantRange) + const localBloom = await this.#algo.bloomFor(id, 0, localWantRange) this.sink.write({ id, phase: 4, @@ -245,13 +254,13 @@ class SyncStream extends Pipeable { * @param {JSON} remoteBloom * @param {Array} msgIDsForMe */ - #sendBloomReq(id, phase, round, remoteBloom, msgIDsForMe) { + async #sendBloomReq(id, phase, round, remoteBloom, msgIDsForMe) { // prettier-ignore this.#debug('%s Stream IN%s: got bloom round %s plus msgIDs in %s: %o', this.#myId, phase-1, round-1, id, msgIDsForMe) const remoteWantRange = this.#remoteWant.get(id) if (!remoteWantRange) throw new Error(`Remote want-range not set for ${id}`) this.#updateReceivableMsgs(id, msgIDsForMe) - const msgIDsForThem = this.#algo.getMsgsMissing( + const msgIDsForThem = await this.#algo.getMsgsMissing( id, round - 1, remoteWantRange, @@ -261,7 +270,7 @@ class SyncStream extends Pipeable { const extras = this.#receivableMsgs.get(id) const localWantRange = this.#localWant.get(id) if (!localWantRange) throw new Error(`Local want-range not set for ${id}`) - const localBloom = this.#algo.bloomFor(id, round, localWantRange, extras) + const localBloom = await this.#algo.bloomFor(id, round, localWantRange, extras) this.sink.write({ id, phase, @@ -278,13 +287,13 @@ class SyncStream extends Pipeable { * @param {JSON} remoteBloom * @param {Array} msgIDsForMe */ - #sendBloomRes(id, phase, round, remoteBloom, msgIDsForMe) { + async #sendBloomRes(id, phase, round, remoteBloom, msgIDsForMe) { // prettier-ignore this.#debug('%s Stream IN%s: got bloom round %s plus msgIDs in %s: %o', this.#myId, phase-1, round, id, msgIDsForMe) const remoteWantRange = this.#remoteWant.get(id) if (!remoteWantRange) throw new Error(`Remote want-range not set for ${id}`) this.#updateReceivableMsgs(id, msgIDsForMe) - const msgIDsForThem = this.#algo.getMsgsMissing( + const msgIDsForThem = await this.#algo.getMsgsMissing( id, round, remoteWantRange, @@ -294,7 +303,7 @@ class SyncStream extends Pipeable { const extras = this.#receivableMsgs.get(id) const localWantRange = this.#localWant.get(id) if (!localWantRange) throw new Error(`Local want-range not set for ${id}`) - const localBloom = this.#algo.bloomFor(id, round, localWantRange, extras) + const localBloom = await this.#algo.bloomFor(id, round, localWantRange, extras) this.sink.write({ id, phase, @@ -310,13 +319,13 @@ class SyncStream extends Pipeable { * @param {JSON} remoteBloom * @param {Array} msgIDsForMe */ - #sendMissingMsgsReq(id, round, remoteBloom, msgIDsForMe) { + async #sendMissingMsgsReq(id, round, remoteBloom, msgIDsForMe) { // prettier-ignore this.#debug('%s Stream IN7: got bloom round %s plus msgIDs in %s: %o', this.#myId, round, id, msgIDsForMe) const remoteWantRange = this.#remoteWant.get(id) if (!remoteWantRange) throw new Error(`Remote want-range not set for ${id}`) this.#updateReceivableMsgs(id, msgIDsForMe) - const msgIDsForThem = this.#algo.getMsgsMissing( + const msgIDsForThem = await this.#algo.getMsgsMissing( id, round, remoteWantRange, @@ -324,13 +333,13 @@ class SyncStream extends Pipeable { ) this.#updateSendableMsgs(id, msgIDsForThem) const msgIDs = this.#sendableMsgs.get(id) ?? new Set() - const tangleMsgs = this.#algo.getTangleMsgs(id, msgIDs) - const accountMsgs = this.#algo.filterAndFetchAccountMsgs(msgIDs) + const tangleMsgs = await this.#algo.getTangleMsgs(id, msgIDs) + const accountMsgs = await this.#algo.filterAndFetchAccountMsgs(msgIDs) const msgs = accountMsgs.concat(tangleMsgs) const extras = this.#receivableMsgs.get(id) const localWantRange = this.#localWant.get(id) if (!localWantRange) throw new Error(`Local want-range not set for ${id}`) - const localBloom = this.#algo.bloomFor(id, round, localWantRange, extras) + const localBloom = await this.#algo.bloomFor(id, round, localWantRange, extras) this.sink.write({ id, phase: 8, @@ -349,12 +358,12 @@ class SyncStream extends Pipeable { * @param {JSON} remoteBloom * @param {Array} msgsForMe */ - #sendMissingMsgsRes(id, round, remoteBloom, msgsForMe) { + async #sendMissingMsgsRes(id, round, remoteBloom, msgsForMe) { // prettier-ignore this.#debug('%s Stream IN8: got bloom round %s plus %s msgs in %s', this.#myId, round, msgsForMe.length, id) const remoteWantRange = this.#remoteWant.get(id) if (!remoteWantRange) throw new Error(`Remote want-range not set for ${id}`) - const msgIDsForThem = this.#algo.getMsgsMissing( + const msgIDsForThem = await this.#algo.getMsgsMissing( id, round, remoteWantRange, @@ -362,8 +371,8 @@ class SyncStream extends Pipeable { ) this.#updateSendableMsgs(id, msgIDsForThem) const msgIDs = this.#sendableMsgs.get(id) ?? new Set() - const tangleMsgs = this.#algo.getTangleMsgs(id, msgIDs) - const accountMsgs = this.#algo.filterAndFetchAccountMsgs(msgIDs) + const tangleMsgs = await this.#algo.getTangleMsgs(id, msgIDs) + const accountMsgs = await this.#algo.filterAndFetchAccountMsgs(msgIDs) const msgs = accountMsgs.concat(tangleMsgs) this.sink.write({ id, phase: 9, payload: msgs }) // prettier-ignore @@ -418,11 +427,11 @@ class SyncStream extends Pipeable { * @param {string} id * @param {Range} remoteWantRange */ - #sendMsgsInRemoteWant(id, remoteWantRange) { + async #sendMsgsInRemoteWant(id, remoteWantRange) { const msgs = [] - const rangeMsgs = this.#algo.getMsgsInRange(id, remoteWantRange) - const tangleMsgs = this.#algo.getTangleMsgs(id, rangeMsgs) - const accountMsgs = this.#algo.getAccountMsgsFor(tangleMsgs) + const rangeMsgs = await this.#algo.getMsgsInRange(id, remoteWantRange) + const tangleMsgs = await this.#algo.getTangleMsgs(id, rangeMsgs) + const accountMsgs = await this.#algo.getAccountMsgsFor(tangleMsgs) for (const msg of accountMsgs) msgs.push(msg) for (const msg of tangleMsgs) msgs.push(msg) const msgIDs = this.#algo.getMsgIDs(msgs) @@ -449,7 +458,9 @@ class SyncStream extends Pipeable { * sink method * @param {Data} data */ + // TODO: hmm can this be async? it's a push-stream which is supposed to have a certain shape. let's see what happens if i keep it sync write(data) { + // prettier-ignore if (!data) return this.#debug('Invalid data from remote peer: missing data') // prettier-ignore if (typeof data !== 'object') return this.#debug('Invalid data from remote peer: not an object') diff --git a/lib/util.js b/lib/util.js index 146edf5..3b2a95d 100644 --- a/lib/util.js +++ b/lib/util.js @@ -1,7 +1,7 @@ const bs58 = require('bs58') /** * @typedef {import('./range').Range} Range - * @typedef {import('ppppp-db/msg-v4').Msg} Msg + * @typedef {import('pzp-db/msg-v4').Msg} Msg */ /** diff --git a/package.json b/package.json index fd35307..45e8d0f 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { - "name": "ppppp-sync", - "version": "1.0.0", - "description": "PPPPP replication using Kleppmann's hash graph sync", + "name": "pzp-sync", + "version": "0.0.1", + "description": "PZP replication using Kleppmann's hash graph sync", "author": "Andre Staltz ", "license": "MIT", - "homepage": "https://github.com/staltz/ppppp-sync", + "homepage": "https://codeberg.org/pzp/pzp-sync", "repository": { "type": "git", - "url": "git@github.com:staltz/ppppp-sync.git" + "url": "git@codeberg.org:pzp/pzp-sync.git" }, "main": "index.js", "files": [ @@ -37,12 +37,12 @@ "@types/pull-stream": "3.6.3", "@types/node": "16.x", "c8": "7", - "ppppp-caps": "github:staltz/ppppp-caps#93fa810b9a40b78aef4872d4c2a8412cccb52929", - "ppppp-db": "github:staltz/ppppp-db#cf1532965ea1d16929ed2291a9b737a4ce74caac", - "ppppp-dict": "github:staltz/ppppp-dict#c40d51be6cb96982b4fe691a292b3c12b6f49a36", - "ppppp-goals": "github:staltz/ppppp-goals#46a8d8889c668cf291607963fd7301f21aa634b5", - "ppppp-keypair": "github:staltz/ppppp-keypair#c33980c580e33f9a35cb0c672b916ec9fe8b4c6d", - "ppppp-set": "github:staltz/ppppp-set#07c3e295b2d09d2d6c3ac6b5b93ad2ea80698452", + "pzp-caps": "^1.0.0", + "pzp-db": "^1.0.1", + "pzp-dict": "^1.0.0", + "pzp-goals": "^1.0.0", + "pzp-keypair": "^1.0.0", + "pzp-set": "^1.0.0", "prettier": "^2.6.2", "pretty-quick": "^3.1.3", "rimraf": "^4.4.0", diff --git a/protospec.md b/protospec.md index e8444fa..596923a 100644 --- a/protospec.md +++ b/protospec.md @@ -122,7 +122,7 @@ tangleSlice(tangleID, msgIDs) -> Array ``` /** - * Receives an Array of PPPPP msgs, validates and persists each in the database. + * Receives an Array of PZP msgs, validates and persists each in the database. */ commit(msgs) -> void ``` diff --git a/test/account-sync.test.js b/test/account-sync.test.js index b8ed967..c99afa6 100644 --- a/test/account-sync.test.js +++ b/test/account-sync.test.js @@ -1,14 +1,18 @@ const test = require('node:test') const assert = require('node:assert') const p = require('util').promisify -const Keypair = require('ppppp-keypair') +const Keypair = require('pzp-keypair') const { createPeer } = require('./util') const aliceKeypair = Keypair.generate('ed25519', 'alice') const bobKeys = Keypair.generate('ed25519', 'bob') -function getAccount(iter) { - return [...iter] +async function getAccount(iter) { + const ary = [] + for await (const it of iter) { + ary.push(it) + } + return ary .filter((m) => m.metadata.account === 'self' && m.data?.action === 'add') .map((m) => m.data.key.bytes) } @@ -40,13 +44,13 @@ test('sync an account tangle', async (t) => { }) assert.deepEqual( - getAccount(alice.db.msgs()), + await getAccount(alice.db.msgs()), [aliceKeypair.public, aliceKeypair1.public, aliceKeypair2.public], 'alice has her account tangle' ) assert.deepEqual( - getAccount(bob.db.msgs()), + await getAccount(bob.db.msgs()), [], "bob doesn't have alice's account tangle" ) @@ -64,7 +68,7 @@ test('sync an account tangle', async (t) => { assert('sync!') assert.deepEqual( - getAccount(bob.db.msgs()), + await getAccount(bob.db.msgs()), [aliceKeypair.public, aliceKeypair1.public, aliceKeypair2.public], "bob has alice's account tangle" ) diff --git a/test/dict-sync.test.js b/test/dict-sync.test.js index d66077f..3effcfd 100644 --- a/test/dict-sync.test.js +++ b/test/dict-sync.test.js @@ -1,12 +1,20 @@ const test = require('node:test') const assert = require('node:assert') const p = require('node:util').promisify -const Keypair = require('ppppp-keypair') -const MsgV4 = require('ppppp-db/msg-v4') +const Keypair = require('pzp-keypair') +const MsgV4 = require('pzp-db/msg-v4') const { createPeer } = require('./util') const aliceKeypair = Keypair.generate('ed25519', 'alice') +async function flatten(iter) { + const ary = [] + for await (const it of iter) { + ary.push(it) + } + return ary +} + test('sync goal=dict from scratch', async (t) => { const SPAN = 5 const alice = createPeer({ @@ -27,7 +35,7 @@ test('sync goal=dict from scratch', async (t) => { _nonce: 'alice', }) await p(alice.dict.load)(aliceID) - const aliceAccountRoot = alice.db.get(aliceID) + const aliceAccountRoot = await p(alice.db.get)(aliceID) // Bob knows Alice await p(bob.db.add)(aliceAccountRoot, aliceID) @@ -39,7 +47,7 @@ test('sync goal=dict from scratch', async (t) => { // Assert situation at Alice before sync { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .map((msg) => msg.data?.update) .filter((x) => !!x) .map((x) => x.age ?? x.name ?? x.gender) @@ -48,7 +56,7 @@ test('sync goal=dict from scratch', async (t) => { // Assert situation at Bob before sync { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .map((msg) => msg.data?.update) .filter((x) => !!x) .map((x) => x.age ?? x.name ?? x.gender) @@ -66,7 +74,7 @@ test('sync goal=dict from scratch', async (t) => { // Assert situation at Bob after sync { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .map((msg) => msg.data?.update) .filter((x) => !!x) .map((x) => x.age ?? x.name ?? x.gender) @@ -104,7 +112,7 @@ test('sync goal=dict with ghostSpan=2', async (t) => { _nonce: 'alice', }) await p(alice.dict.load)(aliceID) - const aliceAccountRoot = alice.db.get(aliceID) + const aliceAccountRoot = await p(alice.db.get)(aliceID) // Bob knows Alice await p(bob.db.add)(aliceAccountRoot, aliceID) @@ -121,7 +129,7 @@ test('sync goal=dict with ghostSpan=2', async (t) => { let rec3 let rec4 let rec5 - for (const rec of alice.db.records()) { + for await (const rec of alice.db.records()) { if (rec.msg.metadata.dataSize === 0) moot = rec if (rec.msg.data?.update?.name === 'alice') rec1 = rec if (rec.msg.data?.update?.age === 24) rec2 = rec @@ -161,7 +169,7 @@ test('sync goal=dict with ghostSpan=2', async (t) => { const fieldRoots = alice.dict._getFieldRoots('profile') assert.deepEqual(fieldRoots.age, [rec4.id]) assert.deepEqual(fieldRoots.name, [rec5.id]) - const tangle = alice.db.getTangle(alice.dict.getFeedID('profile')) + const tangle = await p(alice.db.getTangle)(alice.dict.getFeedID('profile')) const { deletables, erasables } = tangle.getDeletablesAndErasables(rec4.id) assert.equal(deletables.size, 2) assert.equal(erasables.size, 2) @@ -182,7 +190,7 @@ test('sync goal=dict with ghostSpan=2', async (t) => { // Assert situation at Alice before sync assert.deepEqual( - alice.dict.read(aliceID, 'profile'), + await p(alice.dict.read)(aliceID, 'profile'), { age: 25, name: 'ALICE' }, 'alice has age+name dict' ) @@ -199,7 +207,7 @@ test('sync goal=dict with ghostSpan=2', async (t) => { // Assert situation at Alice before sync: she got the branched off msg assert.deepEqual( - alice.dict.read(aliceID, 'profile'), + await p(alice.dict.read)(aliceID, 'profile'), { age: 25, name: 'ALICE', gender: 'w' }, 'alice has age+name+gender dict' ) diff --git a/test/feed-sync.test.js b/test/feed-sync.test.js index 1026f0c..ce37251 100644 --- a/test/feed-sync.test.js +++ b/test/feed-sync.test.js @@ -1,13 +1,21 @@ const test = require('node:test') const assert = require('node:assert') const p = require('node:util').promisify -const Keypair = require('ppppp-keypair') +const Keypair = require('pzp-keypair') const Algorithm = require('../lib/algorithm') const { createPeer } = require('./util') const carolKeypair = Keypair.generate('ed25519', 'carol') const bobKeypair2 = Keypair.generate('ed25519', 'bob2') +async function flatten(iter) { + const ary = [] + for await (const it of iter) { + ary.push(it) + } + return ary +} + test('sync a feed without pre-knowing the owner account', async (t) => { const alice = createPeer({ name: 'alice' }) const bob = createPeer({ name: 'bob' }) @@ -32,7 +40,7 @@ test('sync a feed without pre-knowing the owner account', async (t) => { const bobPostsID = bob.db.feed.getID(bobID, 'post') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'alice has no posts from bob') @@ -49,7 +57,7 @@ test('sync a feed without pre-knowing the owner account', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -88,7 +96,7 @@ test('sync a feed with updated msgs from new account keypair', async (t) => { const bobPostsID = bob.db.feed.getID(bobID, 'post') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'alice has no posts from bob') @@ -105,7 +113,7 @@ test('sync a feed with updated msgs from new account keypair', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -148,7 +156,7 @@ test('sync a feed with updated msgs from new account keypair', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -175,7 +183,7 @@ test('sync a feed with goal=all', async (t) => { subdomain: 'account', _nonce: 'carol', }) - const carolAccountRoot = alice.db.get(carolID) + const carolAccountRoot = await p(alice.db.get)(carolID) // Bob knows Carol await p(bob.db.add)(carolAccountRoot, carolID) @@ -193,7 +201,7 @@ test('sync a feed with goal=all', async (t) => { assert('alice has msgs 1..10 from carol') const carolPostsMootID = alice.db.feed.getID(carolID, 'post') - const carolPostsMoot = alice.db.get(carolPostsMootID) + const carolPostsMoot = await p(alice.db.get)(carolPostsMootID) await p(bob.db.add)(carolPostsMoot, carolPostsMootID) for (let i = 0; i < 7; i++) { @@ -201,7 +209,7 @@ test('sync a feed with goal=all', async (t) => { } { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -222,7 +230,7 @@ test('sync a feed with goal=all', async (t) => { assert('sync!') { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -249,7 +257,7 @@ test('sync a feed with goal=newest', async (t) => { subdomain: 'account', _nonce: 'carol', }) - const carolAccountRoot = alice.db.get(carolID) + const carolAccountRoot = await p(alice.db.get)(carolID) // Bob knows Carol await p(bob.db.add)(carolAccountRoot, carolID) @@ -267,7 +275,7 @@ test('sync a feed with goal=newest', async (t) => { assert('alice has msgs 1..10 from carol') const carolPostsMootID = alice.db.feed.getID(carolID, 'post') - const carolPostsMoot = alice.db.get(carolPostsMootID) + const carolPostsMoot = await p(alice.db.get)(carolPostsMootID) await p(bob.db.add)(carolPostsMoot, carolPostsMootID) for (let i = 0; i < 7; i++) { @@ -275,7 +283,7 @@ test('sync a feed with goal=newest', async (t) => { } { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -296,7 +304,7 @@ test('sync a feed with goal=newest', async (t) => { assert('sync!') { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -323,7 +331,7 @@ test('sync a feed with goal=newest but too far behind', async (t) => { subdomain: 'account', _nonce: 'carol', }) - const carolIDMsg = alice.db.get(carolID) + const carolIDMsg = await p(alice.db.get)(carolID) // Bob knows Carol await p(bob.db.add)(carolIDMsg, carolID) @@ -340,12 +348,12 @@ test('sync a feed with goal=newest but too far behind', async (t) => { } const carolPostsMootID = alice.db.feed.getID(carolID, 'post') - const carolPostsMoot = alice.db.get(carolPostsMootID) + const carolPostsMoot = await p(alice.db.get)(carolPostsMootID) const algo = new Algorithm(alice) await algo.pruneNewest(carolPostsMootID, 5) { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -361,7 +369,7 @@ test('sync a feed with goal=newest but too far behind', async (t) => { } { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, ['m1', 'm2'], 'bob has msgs 1..2 from carol') @@ -378,7 +386,7 @@ test('sync a feed with goal=newest but too far behind', async (t) => { assert('sync!') { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -409,7 +417,7 @@ test('sync small newest slice of a feed, then the whole feed', async (t) => { subdomain: 'account', _nonce: 'carol', }) - const carolIDMsg = carol.db.get(carolID) + const carolIDMsg = await p(carol.db.get)(carolID) // Alice and Bob know Carol await p(alice.db.add)(carolIDMsg, carolID) @@ -426,17 +434,17 @@ test('sync small newest slice of a feed, then the whole feed', async (t) => { } const carolPostsMootID = carol.db.feed.getID(carolID, 'post') - const carolPostsMoot = carol.db.get(carolPostsMootID) + const carolPostsMoot = await p(carol.db.get)(carolPostsMootID) { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'bob has nothing from carol') } { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'alice has nothing from carol') @@ -454,7 +462,7 @@ test('sync small newest slice of a feed, then the whole feed', async (t) => { assert('sync!') { - const arr = [...bob.db.msgs()] + const arr = (await flatten(bob.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -474,7 +482,7 @@ test('sync small newest slice of a feed, then the whole feed', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) assert.deepEqual( @@ -494,7 +502,7 @@ test('sync small newest slice of a feed, then the whole feed', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === carolID && msg.data) .map((msg) => msg.data.text) .sort() diff --git a/test/realtime.test.js b/test/realtime.test.js index 3e9eca0..3dc3b12 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -3,6 +3,14 @@ const assert = require('node:assert') const p = require('node:util').promisify const { createPeer } = require('./util') +async function flatten(iter) { + const ary = [] + for await (const it of iter) { + ary.push(it) + } + return ary +} + test('sync feed msgs in realtime after the 9 rounds', async (t) => { const alice = createPeer({ name: 'alice' }) const bob = createPeer({ name: 'bob' }) @@ -25,7 +33,7 @@ test('sync feed msgs in realtime after the 9 rounds', async (t) => { const bobPostsID = bob.db.feed.getID(bobID, 'post') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'alice has no posts from bob') @@ -42,7 +50,7 @@ test('sync feed msgs in realtime after the 9 rounds', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, ['m0'], 'alice has post 0 from bob') @@ -65,7 +73,7 @@ test('sync feed msgs in realtime after the 9 rounds', async (t) => { { let arr for (let i = 0; i < 100; i++) { - arr = [...alice.db.msgs()] + arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) if (arr.length < 3) { @@ -103,7 +111,7 @@ test('sync feed msgs in realtime after the 9 rounds, reverse', async (t) => { const bobPostsID = bob.db.feed.getID(bobID, 'post') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, [], 'alice has no posts from bob') @@ -121,7 +129,7 @@ test('sync feed msgs in realtime after the 9 rounds, reverse', async (t) => { assert('sync!') { - const arr = [...alice.db.msgs()] + const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) assert.deepEqual(arr, ['m0'], 'alice has post 0 from bob') @@ -144,7 +152,7 @@ test('sync feed msgs in realtime after the 9 rounds, reverse', async (t) => { { let arr for (let i = 0; i < 100; i++) { - arr = [...alice.db.msgs()] + arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) .map((msg) => msg.data.text) if (arr.length < 3) { diff --git a/test/set-sync.test.js b/test/set-sync.test.js index 2ef4878..94c2540 100644 --- a/test/set-sync.test.js +++ b/test/set-sync.test.js @@ -1,8 +1,8 @@ const test = require('node:test') const assert = require('node:assert') const p = require('node:util').promisify -const Keypair = require('ppppp-keypair') -const MsgV4 = require('ppppp-db/msg-v4') +const Keypair = require('pzp-keypair') +const MsgV4 = require('pzp-db/msg-v4') const { createPeer } = require('./util') const aliceKeypair = Keypair.generate('ed25519', 'alice') @@ -35,7 +35,7 @@ test('sync goal=set from scratch', async (t) => { _nonce: 'alice', }) await p(alice.set.load)(aliceID) - const aliceAccountRoot = alice.db.get(aliceID) + const aliceAccountRoot = await p(alice.db.get)(aliceID) // Bob knows Alice const bobID = await p(bob.db.account.create)({ @@ -52,13 +52,13 @@ test('sync goal=set from scratch', async (t) => { // Assert situation at Alice before sync assert.deepEqual( - alice.set.values('names', aliceID), + await p(alice.set.values)('names', aliceID), ['Alice', 'Bob'], 'alice has Alice+Bob set' ) // Assert situation at Bob before sync - assert.deepEqual(bob.set.values('names', aliceID), [], 'bob has empty set') + assert.deepEqual(await p(bob.set.values)('names', aliceID), [], 'bob has empty set') // Trigger sync alice.goals.set(mootID, 'set') @@ -71,7 +71,7 @@ test('sync goal=set from scratch', async (t) => { // Assert situation at Bob after sync assert.deepEqual( - bob.set.values('names', aliceID), + await p(bob.set.values)('names', aliceID), ['Alice', 'Bob'], 'bob has Alice+Bob set' ) @@ -107,7 +107,7 @@ test('sync goal=set with ghostSpan=5', async (t) => { _nonce: 'alice', }) await p(alice.set.load)(aliceID) - const aliceAccountRoot = alice.db.get(aliceID) + const aliceAccountRoot = await p(alice.db.get)(aliceID) // Bob knows Alice await p(bob.db.add)(aliceAccountRoot, aliceID) @@ -126,7 +126,7 @@ test('sync goal=set with ghostSpan=5', async (t) => { let rec4 let rec5 let rec6 - for (const rec of alice.db.records()) { + for await (const rec of alice.db.records()) { if (rec.msg.metadata.dataSize === 0) moot = rec if (rec.msg.data?.add?.[0] === 'alice') rec1 = rec if (rec.msg.data?.add?.[0] === 'bob') rec2 = rec @@ -167,7 +167,7 @@ test('sync goal=set with ghostSpan=5', async (t) => { { const itemRoots = alice.set._getItemRoots('follows') assert.deepEqual(itemRoots, { Alice: [rec5.id], Bob: [rec6.id] }) - const tangle = alice.db.getTangle(alice.set.getFeedID('follows')) + const tangle = await p(alice.db.getTangle)(alice.set.getFeedID('follows')) const { deletables, erasables } = tangle.getDeletablesAndErasables(rec5.id) assert.equal(deletables.size, 2) assert.equal(erasables.size, 3) @@ -189,7 +189,7 @@ test('sync goal=set with ghostSpan=5', async (t) => { // Assert situation at Alice before sync assert.deepEqual( - alice.set.values('follows', aliceID), + await p(alice.set.values)('follows', aliceID), ['Alice', 'Bob'], 'alice has Alice+Bob set' ) @@ -206,7 +206,7 @@ test('sync goal=set with ghostSpan=5', async (t) => { // Assert situation at Alice after sync: she got the branched off msg assert.deepEqual( - alice.set.values('follows', aliceID), + await p(alice.set.values)('follows', aliceID), ['Carol', 'Alice', 'Bob'], 'alice has Alice+Bob+Carol set' ) diff --git a/test/thread-sync.test.js b/test/thread-sync.test.js index ef9a54c..08d17ce 100644 --- a/test/thread-sync.test.js +++ b/test/thread-sync.test.js @@ -1,14 +1,18 @@ const test = require('node:test') const assert = require('node:assert') const p = require('util').promisify -const Keypair = require('ppppp-keypair') +const Keypair = require('pzp-keypair') const { createPeer } = require('./util') const carolKeypair = Keypair.generate('ed25519', 'carol') const daveKeypair = Keypair.generate('ed25519', 'dave') -function getTexts(iter) { - return [...iter].filter((msg) => msg.data?.text).map((msg) => msg.data.text) +async function getTexts(iter) { + const ary = [] + for await (i of iter) { + ary.push(i) + } + return ary.filter((msg) => msg.data?.text).map((msg) => msg.data.text) } /* @@ -57,14 +61,14 @@ test('sync a thread where both peers have portions', async (t) => { subdomain: 'account', _nonce: 'alice', }) - const aliceIDMsg = alice.db.get(aliceID) + const aliceIDMsg = await p(alice.db.get)(aliceID) await bob.db.loaded() const bobID = await p(bob.db.account.create)({ subdomain: 'account', _nonce: 'bob', }) - const bobIDMsg = bob.db.get(bobID) + const bobIDMsg = await p(bob.db.get)(bobID) // Alice created Carol const carolID = await p(alice.db.account.create)({ @@ -72,7 +76,7 @@ test('sync a thread where both peers have portions', async (t) => { keypair: carolKeypair, _nonce: 'carol', }) - const carolIDMsg = alice.db.get(carolID) + const carolIDMsg = await p(alice.db.get)(carolID) // Alice created Dave const daveID = await p(alice.db.account.create)({ @@ -80,7 +84,7 @@ test('sync a thread where both peers have portions', async (t) => { keypair: daveKeypair, _nonce: 'dave', }) - const daveIDMsg = alice.db.get(daveID) + const daveIDMsg = await p(alice.db.get)(daveID) // Alice knows Bob await p(alice.db.add)(bobIDMsg, bobID) @@ -96,7 +100,7 @@ test('sync a thread where both peers have portions', async (t) => { data: { text: 'A' }, }) const rootHashA = alice.db.feed.getID(aliceID, 'post') - const rootMsgA = alice.db.get(rootHashA) + const rootMsgA = await p(alice.db.get)(rootHashA) await p(bob.db.add)(rootMsgA, rootHashA) await p(bob.db.add)(startA.msg, rootHashA) @@ -115,7 +119,7 @@ test('sync a thread where both peers have portions', async (t) => { tangles: [startA.id], }) const rootHashB = bob.db.feed.getID(bobID, 'post') - const rootMsgB = bob.db.get(rootHashB) + const rootMsgB = await p(bob.db.get)(rootHashB) await p(alice.db.add)(rootMsgB, rootHashB) await p(alice.db.add)(replyB1.msg, rootHashB) @@ -138,13 +142,13 @@ test('sync a thread where both peers have portions', async (t) => { }) assert.deepEqual( - getTexts(alice.db.msgs()), + await getTexts(alice.db.msgs()), ['A', 'B1', 'B2', 'C1'], 'alice has a portion of the thread' ) assert.deepEqual( - getTexts(bob.db.msgs()), + await getTexts(bob.db.msgs()), ['A', 'B1', 'B2', 'D1'], 'bob has another portion of the thread' ) @@ -160,13 +164,13 @@ test('sync a thread where both peers have portions', async (t) => { assert('sync!') assert.deepEqual( - getTexts(alice.db.msgs()), + await getTexts(alice.db.msgs()), ['A', 'B1', 'B2', 'C1', 'D1'], 'alice has the full thread' ) assert.deepEqual( - getTexts(bob.db.msgs()), + await getTexts(bob.db.msgs()), ['A', 'B1', 'B2', 'D1', 'C1'], 'bob has the full thread' ) @@ -185,14 +189,14 @@ test('sync a thread where initiator does not have the root', async (t) => { subdomain: 'account', _nonce: 'alice', }) - const aliceIDMsg = alice.db.get(aliceID) + const aliceIDMsg = await p(alice.db.get)(aliceID) await bob.db.loaded() const bobID = await p(bob.db.account.create)({ subdomain: 'account', _nonce: 'bob', }) - const bobIDMsg = bob.db.get(bobID) + const bobIDMsg = await p(bob.db.get)(bobID) // Alice knows Bob await p(alice.db.add)(bobIDMsg, bobID) @@ -221,12 +225,12 @@ test('sync a thread where initiator does not have the root', async (t) => { }) assert.deepEqual( - getTexts(alice.db.msgs()), + await getTexts(alice.db.msgs()), ['A', 'A1', 'A2'], 'alice has the full thread' ) - assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') + assert.deepEqual(await getTexts(bob.db.msgs()), [], 'bob has nothing') bob.goals.set(rootA.id, 'all') // ON PURPOSE: alice does not set the goal @@ -240,7 +244,7 @@ test('sync a thread where initiator does not have the root', async (t) => { assert('sync!') assert.deepEqual( - getTexts(bob.db.msgs()), + await getTexts(bob.db.msgs()), ['A', 'A1', 'A2'], 'bob has the full thread' ) @@ -259,14 +263,14 @@ test('sync a thread where receiver does not have the root', async (t) => { subdomain: 'account', _nonce: 'alice', }) - const aliceIDMsg = alice.db.get(aliceID) + const aliceIDMsg = await p(alice.db.get)(aliceID) await bob.db.loaded() const bobID = await p(bob.db.account.create)({ subdomain: 'account', _nonce: 'bob', }) - const bobIDMsg = bob.db.get(bobID) + const bobIDMsg = await p(bob.db.get)(bobID) // Alice knows Bob await p(alice.db.add)(bobIDMsg, bobID) @@ -295,12 +299,12 @@ test('sync a thread where receiver does not have the root', async (t) => { }) assert.deepEqual( - getTexts(alice.db.msgs()), + await getTexts(alice.db.msgs()), ['A', 'A1', 'A2'], 'alice has the full thread' ) - assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') + assert.deepEqual(await getTexts(bob.db.msgs()), [], 'bob has nothing') bob.goals.set(rootA.id, 'all') alice.goals.set(rootA.id, 'all') @@ -313,7 +317,7 @@ test('sync a thread where receiver does not have the root', async (t) => { assert('sync!') assert.deepEqual( - getTexts(bob.db.msgs()), + await getTexts(bob.db.msgs()), ['A', 'A1', 'A2'], 'bob has the full thread' ) @@ -332,14 +336,14 @@ test('sync a thread with reactions too', async (t) => { subdomain: 'account', _nonce: 'alice', }) - const aliceIDMsg = alice.db.get(aliceID) + const aliceIDMsg = await p(alice.db.get)(aliceID) await bob.db.loaded() const bobID = await p(bob.db.account.create)({ subdomain: 'account', _nonce: 'bob', }) - const bobIDMsg = bob.db.get(bobID) + const bobIDMsg = await p(bob.db.get)(bobID) // Alice knows Bob await p(alice.db.add)(bobIDMsg, bobID) @@ -375,12 +379,12 @@ test('sync a thread with reactions too', async (t) => { }) assert.deepEqual( - getTexts(alice.db.msgs()), + await getTexts(alice.db.msgs()), ['A', 'A1', 'A2', 'yes'], 'alice has the full thread' ) - assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') + assert.deepEqual(await getTexts(bob.db.msgs()), [], 'bob has nothing') bob.goals.set(rootA.id, 'all') alice.goals.set(rootA.id, 'all') @@ -393,7 +397,7 @@ test('sync a thread with reactions too', async (t) => { assert('sync!') assert.deepEqual( - getTexts(bob.db.msgs()), + await getTexts(bob.db.msgs()), ['A', 'A1', 'A2', 'yes'], 'bob has the full thread' ) diff --git a/test/util.js b/test/util.js index 3225112..465faea 100644 --- a/test/util.js +++ b/test/util.js @@ -1,15 +1,15 @@ const OS = require('node:os') const Path = require('node:path') const rimraf = require('rimraf') -const caps = require('ppppp-caps') -const Keypair = require('ppppp-keypair') +const caps = require('pzp-caps') +const Keypair = require('pzp-keypair') function createPeer(config) { if (config.name) { const name = config.name const tmp = OS.tmpdir() config.global ??= {} - config.global.path ??= Path.join(tmp, `ppppp-sync-${name}-${Date.now()}`) + config.global.path ??= Path.join(tmp, `pzp-sync-${name}-${Date.now()}`) config.global.keypair ??= Keypair.generate('ed25519', name) delete config.name } @@ -27,10 +27,10 @@ function createPeer(config) { return require('secret-stack/bare')() .use(require('secret-stack/plugins/net')) .use(require('secret-handshake-ext/secret-stack')) - .use(require('ppppp-db')) - .use(require('ppppp-dict')) - .use(require('ppppp-set')) - .use(require('ppppp-goals')) + .use(require('pzp-db')) + .use(require('pzp-dict')) + .use(require('pzp-set')) + .use(require('pzp-goals')) .use(require('ssb-box')) .use(require('../lib')) .call(null, { diff --git a/test/want-range.test.js b/test/want-range.test.js index 77bfbc3..c68e98c 100644 --- a/test/want-range.test.js +++ b/test/want-range.test.js @@ -4,41 +4,41 @@ const Algorithm = require('../lib/algorithm') const EMPTY = [1, 0] -test('want-range for goal=newest-3', (t) => { +test('want-range for goal=newest-3', async (t) => { const algo = new Algorithm({ db: null }) const goal = { type: 'newest', count: 3 } - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 3], goal), [2, 3]) - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 5], goal), [3, 5]) - assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [2, 4]) - assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [3, 4]) - assert.deepStrictEqual(algo.wantRange([1, 3], [4, 6], goal), [4, 6]) - assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), EMPTY) - assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 3], goal), [2, 3]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 5], goal), [3, 5]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [2, 4], goal), [2, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 5], [2, 4], goal), [3, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [4, 6], goal), [4, 6]) + assert.deepStrictEqual(await algo.wantRange([4, 6], [1, 3], goal), EMPTY) + assert.deepStrictEqual(await algo.wantRange([1, 3], [6, 7], goal), [6, 7]) }) -test('want-range for goal=all', (t) => { +test('want-range for goal=all', async (t) => { const algo = new Algorithm({ db: null }) const goal = { type: 'all' } - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 3], goal), [1, 3]) - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 5], goal), [1, 5]) - assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [2, 4]) - assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [2, 4]) - assert.deepStrictEqual(algo.wantRange([1, 3], [4, 6], goal), [4, 6]) - assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), [1, 3]) - assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 3], goal), [1, 3]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 5], goal), [1, 5]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [2, 4], goal), [2, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 5], [2, 4], goal), [2, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [4, 6], goal), [4, 6]) + assert.deepStrictEqual(await algo.wantRange([4, 6], [1, 3], goal), [1, 3]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [6, 7], goal), [6, 7]) }) -test('want-range for goal=dict', (t) => { - const algo = new Algorithm({ db: null, dict: { minGhostDepth: () => 3 } }) +test('want-range for goal=dict', async (t) => { + const algo = new Algorithm({ db: null, dict: { minGhostDepth: (id, cb) => cb(null, 3) } }) const goal = { type: 'dict' } - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 3], goal), [3, 3]) - assert.deepStrictEqual(algo.wantRange([2, 4], [1, 5], goal), [3, 5]) - assert.deepStrictEqual(algo.wantRange([1, 3], [2, 4], goal), [3, 4]) - assert.deepStrictEqual(algo.wantRange([1, 5], [2, 4], goal), [3, 4]) - assert.deepStrictEqual(algo.wantRange([1, 3], [4, 6], goal), [4, 6]) - assert.deepStrictEqual(algo.wantRange([4, 6], [1, 3], goal), [3, 3]) - assert.deepStrictEqual(algo.wantRange([1, 3], [6, 7], goal), [6, 7]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 3], goal), [3, 3]) + assert.deepStrictEqual(await algo.wantRange([2, 4], [1, 5], goal), [3, 5]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [2, 4], goal), [3, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 5], [2, 4], goal), [3, 4]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [4, 6], goal), [4, 6]) + assert.deepStrictEqual(await algo.wantRange([4, 6], [1, 3], goal), [3, 3]) + assert.deepStrictEqual(await algo.wantRange([1, 3], [6, 7], goal), [6, 7]) })