From 043faed83680a1292a63bf3aedcf410698192025 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 15:14:05 +0200 Subject: [PATCH 1/7] Import realtime test from sync module --- test/realtime.test.js | 98 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 test/realtime.test.js diff --git a/test/realtime.test.js b/test/realtime.test.js new file mode 100644 index 0000000..95e963d --- /dev/null +++ b/test/realtime.test.js @@ -0,0 +1,98 @@ +const test = require('node:test') +const assert = require('node:assert') +const p = require('node:util').promisify +const { createPeer } = require('./util') + +async function flatten(iter) { + const ary = [] + for await (const it of iter) { + ary.push(it) + } + return ary +} + +// copy of test in pzp-sync, but that doesn't test if it works with gc enabled +test('create 200 messages that manage to replicate with low "newest" goals', async (t) => { + const n = 200 + + const alice = createPeer({ name: 'alice' }) + const bob = createPeer({ name: 'bob' }) + + await alice.db.loaded() + await bob.db.loaded() + + const aliceID = await p(alice.db.account.create)({ + subdomain: 'account', + _nonce: 'alice', + }) + const bobID = await p(bob.db.account.create)({ + subdomain: 'account', + _nonce: 'bob', + }) + + const bobPostsID = bob.db.feed.getID(bobID, 'post') + + { + const arr = (await flatten(alice.db.msgs())) + .filter((msg) => msg.metadata.account === bobID && msg.data) + .map((msg) => msg.data.text) + assert.deepEqual(arr, [], 'alice has no posts from bob') + } + + const confirmed = [] + // for keeping track of which msgs have arrived + for (let i = 0; i < n; i++) { + confirmed.push(false) + } + + alice.db.onRecordAdded(rec => { + if (rec.msg.data?.text) { + const num = Number.parseInt(rec.msg.data.text) + confirmed[num] = true + } + }) + + await p(alice.set.load)(aliceID) + await p(bob.set.load)(bobID) + + // TODO: do we need to follow each other? + assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') + + //bob.goals.set(bobPostsID, 'newest-50') + //alice.goals.set(bobPostsID, 'newest-50') + // TODO: lower to newest-50, alice then bob + // supposedly [myrules, theirrules], do they need to match? + await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 64_000_000) + await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 64_000_000) + + const remoteAlice = await p(bob.connect)(alice.getAddress()) + assert('alice and bob connected') + + bob.sync.start() + await p(setTimeout)(1000) + assert('sync!') + + const hundred = [] + for (let i = 0; i < n; i++) { + hundred.push(i) + } + Promise.all(hundred.map(i => p(bob.db.feed.publish)({ + account: bobID, + domain: 'post', + data: { text: `${i}` }, + }))) + assert(`bob published ${n} posts in parallel`) + + //let tries = 30 + let tries = 100 + // just waiting for them to arrive + do { + await p(setTimeout)(100) + } while (!confirmed.every(v => v === true) && tries-- > 0) + + assert.equal(confirmed.filter(v => v === true).length, n, `alice has all of bob's posts`) + + await p(remoteAlice.close)(true) + await p(alice.close)(true) + await p(bob.close)(true) +}) \ No newline at end of file From 451975eca49493f9c42cc97a7c3a6d7e6257af25 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 17:28:49 +0200 Subject: [PATCH 2/7] Get realtime partial to work with initial post --- lib/index.js | 5 +++++ package.json | 12 ++++++------ test/realtime.test.js | 26 +++++++++++++++----------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/index.js b/lib/index.js index 13d3e3c..eef3c6e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,6 +2,7 @@ const makeDebug = require('debug') const MsgV4 = require('pzp-db/msg-v4') /** + * @typedef {import('pzp-db').RecPresent} Rec * @typedef {ReturnType} PZPDB * @typedef {ReturnType} PZPGoal * @typedef {import('pzp-goals').GoalDSL} GoalDSL @@ -276,6 +277,10 @@ function initConductor(peer, config) { peer.gc.start(maxBytes) peer.sync.start() + peer.db.onRecordAdded((/** @type {Rec} */ { id: msgID, msg }) => { + peer.sync.start() + }) + cb() }) } diff --git a/package.json b/package.json index 3e1fcdd..69dd79e 100644 --- a/package.json +++ b/package.json @@ -31,17 +31,17 @@ "bs58": "^5.0.0", "c8": "7", "pzp-caps": "^1.0.0", - "pzp-db": "^1.0.1", - "pzp-dict": "^1.0.0", + "pzp-db": "^1.0.4", + "pzp-dict": "^1.0.1", "pzp-gc": "^1.0.0", - "pzp-goals": "^1.0.0", + "pzp-goals": "^1.0.1", "pzp-keypair": "^1.0.0", - "pzp-set": "^1.0.0", - "pzp-sync": "^1.0.0", + "pzp-set": "^1.0.1", + "pzp-sync": "^1.0.4", "prettier": "^2.6.2", "pretty-quick": "^3.1.3", "rimraf": "^4.4.0", - "secret-handshake-ext": "~0.0.11", + "secret-handshake-ext": "~0.0.12", "secret-stack": "~8.1.0", "ssb-box": "^1.0.1", "typescript": "^5.4.5" diff --git a/test/realtime.test.js b/test/realtime.test.js index 95e963d..31bfcc5 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -39,6 +39,13 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy assert.deepEqual(arr, [], 'alice has no posts from bob') } + // TODO: make things work with this first, then add another test without this? + await p(bob.db.feed.publish)({ + account: bobID, + domain: 'post', + data: { text: `${n}` }, + }) + const confirmed = [] // for keeping track of which msgs have arrived for (let i = 0; i < n; i++) { @@ -47,6 +54,7 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy alice.db.onRecordAdded(rec => { if (rec.msg.data?.text) { + //console.log('alice received rec', rec) const num = Number.parseInt(rec.msg.data.text) confirmed[num] = true } @@ -55,20 +63,17 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy await p(alice.set.load)(aliceID) await p(bob.set.load)(bobID) - // TODO: do we need to follow each other? + // TODO: do we need to follow? probably assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') + // TODO: remove this one + assert(await p(bob.set.add)('follows', aliceID), 'alice follows bob') - //bob.goals.set(bobPostsID, 'newest-50') - //alice.goals.set(bobPostsID, 'newest-50') - // TODO: lower to newest-50, alice then bob - // supposedly [myrules, theirrules], do they need to match? - await p(alice.conductor.start)(aliceID, [['post@all'], ['post@all']], 64_000_000) - await p(bob.conductor.start)(bobID, [['post@all'], ['post@all']], 64_000_000) + await p(alice.conductor.start)(aliceID, [['post@newest-50'], ['post@newest-50']], 64_000_000) + await p(bob.conductor.start)(bobID, [['post@newest-50'], ['post@newest-50']], 64_000_000) const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('alice and bob connected') - bob.sync.start() await p(setTimeout)(1000) assert('sync!') @@ -83,14 +88,13 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy }))) assert(`bob published ${n} posts in parallel`) - //let tries = 30 - let tries = 100 + let tries = 30 // just waiting for them to arrive do { await p(setTimeout)(100) } while (!confirmed.every(v => v === true) && tries-- > 0) - assert.equal(confirmed.filter(v => v === true).length, n, `alice has all of bob's posts`) + assert.equal(confirmed.filter(v => v === true).length, n + 1, `alice has all of bob's posts including the initial one`) await p(remoteAlice.close)(true) await p(alice.close)(true) From e16aa8b2a974bf6ec38a3e23e877d19c2de95f30 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 17:29:40 +0200 Subject: [PATCH 3/7] Remove index.js hack --- lib/index.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/index.js b/lib/index.js index eef3c6e..13d3e3c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,7 +2,6 @@ const makeDebug = require('debug') const MsgV4 = require('pzp-db/msg-v4') /** - * @typedef {import('pzp-db').RecPresent} Rec * @typedef {ReturnType} PZPDB * @typedef {ReturnType} PZPGoal * @typedef {import('pzp-goals').GoalDSL} GoalDSL @@ -277,10 +276,6 @@ function initConductor(peer, config) { peer.gc.start(maxBytes) peer.sync.start() - peer.db.onRecordAdded((/** @type {Rec} */ { id: msgID, msg }) => { - peer.sync.start() - }) - cb() }) } From e86bcda7348da0ad3a9d5797094b79f007a8875a Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 17:32:50 +0200 Subject: [PATCH 4/7] Switch to 'newest-1' --- test/realtime.test.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/realtime.test.js b/test/realtime.test.js index 31bfcc5..dc6ef29 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -63,13 +63,11 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy await p(alice.set.load)(aliceID) await p(bob.set.load)(bobID) - // TODO: do we need to follow? probably assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') - // TODO: remove this one - assert(await p(bob.set.add)('follows', aliceID), 'alice follows bob') - await p(alice.conductor.start)(aliceID, [['post@newest-50'], ['post@newest-50']], 64_000_000) - await p(bob.conductor.start)(bobID, [['post@newest-50'], ['post@newest-50']], 64_000_000) + const goal = 'post@newest-1' + await p(alice.conductor.start)(aliceID, [[goal], [goal]], 64_000_000) + await p(bob.conductor.start)(bobID, [[goal], [goal]], 64_000_000) const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('alice and bob connected') From f5d6aa7ac0889bb19f3f92a33a000d00f8a4457b Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 17:44:00 +0200 Subject: [PATCH 5/7] Copy working test from sync --- test/realtime.test.js | 75 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/realtime.test.js b/test/realtime.test.js index dc6ef29..22d3346 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -94,6 +94,81 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy assert.equal(confirmed.filter(v => v === true).length, n + 1, `alice has all of bob's posts including the initial one`) + await p(remoteAlice.close)(true) + await p(alice.close)(true) + await p(bob.close)(true) +}) + +test('create 100 messages in parallel that still manage to sync realtime (without creating an initial post before starting realtime)', async (t) => { + const alice = createPeer({ name: 'alice' }) + const bob = createPeer({ name: 'bob' }) + + await alice.db.loaded() + await bob.db.loaded() + + const bobID = await p(bob.db.account.create)({ + subdomain: 'account', + _nonce: 'bob', + }) + + const bobPostsID = bob.db.feed.getID(bobID, 'post') + + // TODO: remove + //await p(bob.db.feed.publish)({ + // account: bobID, + // domain: 'post', + // data: { text: `${n}` }, + //}) + + { + const arr = (await flatten(alice.db.msgs())) + .filter((msg) => msg.metadata.account === bobID && msg.data) + .map((msg) => msg.data.text) + assert.deepEqual(arr, [], 'alice has no posts from bob') + } + + bob.goals.set(bobPostsID, 'all') + alice.goals.set(bobPostsID, 'all') + + const remoteAlice = await p(bob.connect)(alice.getAddress()) + assert('alice and bob connected') + + bob.sync.start() + await p(setTimeout)(1000) + assert('sync!') + + const n = 100 + const hundred = [] + for (let i = 0; i < n; i++) { + hundred.push(i) + } + await Promise.all(hundred.map(i => p(bob.db.feed.publish)({ + account: bobID, + domain: 'post', + data: { text: `post nr ${i}` }, + }))) + assert('bob published 100 posts in parallel') + + const bobMsgs = await flatten(bob.db.msgs()) + // 1 for creating bob's account, and 1 for the 'post' moot + assert.equal(bobMsgs.length, n + 2, "bob has all of his own messages") + + let arr + // just waiting for them to arrive + for (let i = 0; i < 100; i++) { + arr = (await flatten(alice.db.msgs())) + // moot doesn't have msg.data + .filter((msg) => msg.metadata.account === bobID && msg.data) + .filter(msg => msg.metadata.domain === 'post') + .map((msg) => msg.data.text) + if (arr.length < n) { + await p(setTimeout)(100) + } else { + break + } + } + assert.equal(arr.length, n, `alice has ${arr.length} posts from bob`) + await p(remoteAlice.close)(true) await p(alice.close)(true) await p(bob.close)(true) From d22a4d106eefe57861693f55bdf8021a555b433e Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Fri, 28 Jun 2024 17:57:54 +0200 Subject: [PATCH 6/7] Add realtime test without initial post --- test/realtime.test.js | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/realtime.test.js b/test/realtime.test.js index 22d3346..8484c08 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -30,8 +30,6 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy _nonce: 'bob', }) - const bobPostsID = bob.db.feed.getID(bobID, 'post') - { const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) @@ -54,7 +52,6 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy alice.db.onRecordAdded(rec => { if (rec.msg.data?.text) { - //console.log('alice received rec', rec) const num = Number.parseInt(rec.msg.data.text) confirmed[num] = true } @@ -106,20 +103,15 @@ test('create 100 messages in parallel that still manage to sync realtime (withou await alice.db.loaded() await bob.db.loaded() + const aliceID = await p(alice.db.account.create)({ + subdomain: 'account', + _nonce: 'alice', + }) const bobID = await p(bob.db.account.create)({ subdomain: 'account', _nonce: 'bob', }) - const bobPostsID = bob.db.feed.getID(bobID, 'post') - - // TODO: remove - //await p(bob.db.feed.publish)({ - // account: bobID, - // domain: 'post', - // data: { text: `${n}` }, - //}) - { const arr = (await flatten(alice.db.msgs())) .filter((msg) => msg.metadata.account === bobID && msg.data) @@ -127,13 +119,18 @@ test('create 100 messages in parallel that still manage to sync realtime (withou assert.deepEqual(arr, [], 'alice has no posts from bob') } - bob.goals.set(bobPostsID, 'all') - alice.goals.set(bobPostsID, 'all') + await p(alice.set.load)(aliceID) + await p(bob.set.load)(bobID) + + assert(await p(alice.set.add)('follows', bobID), 'alice follows bob') + + const goal = 'post@newest-1' + await p(alice.conductor.start)(aliceID, [[goal], [goal]], 64_000_000) + await p(bob.conductor.start)(bobID, [[goal], [goal]], 64_000_000) const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('alice and bob connected') - bob.sync.start() await p(setTimeout)(1000) assert('sync!') From 8c3c92ed530a9dd9c218c475a7fe797bff18ddf7 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Sun, 30 Jun 2024 13:32:12 +0200 Subject: [PATCH 7/7] Remove todo comment in favor of issue --- test/realtime.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/realtime.test.js b/test/realtime.test.js index 8484c08..b196237 100644 --- a/test/realtime.test.js +++ b/test/realtime.test.js @@ -12,7 +12,7 @@ async function flatten(iter) { } // copy of test in pzp-sync, but that doesn't test if it works with gc enabled -test('create 200 messages that manage to replicate with low "newest" goals', async (t) => { +test('create 200 messages that manage to replicate with low "newest" goals (using onRecordAdded and with initial message)', async (t) => { const n = 200 const alice = createPeer({ name: 'alice' }) @@ -37,7 +37,6 @@ test('create 200 messages that manage to replicate with low "newest" goals', asy assert.deepEqual(arr, [], 'alice has no posts from bob') } - // TODO: make things work with this first, then add another test without this? await p(bob.db.feed.publish)({ account: bobID, domain: 'post',