Merge pull request 'Test realtime replicating a lot of messages with a low newest-x goal' (#3) from realtime-partial into master

Reviewed-on: https://codeberg.org/pzp/pzp-conductor/pulls/3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pzp-conductor",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "PZP manager that sets tangle goals",
   "author": "Andre Staltz <contact@staltz.com>",
   "license": "MIT",
@@ -12,7 +12,7 @@
   "type": "commonjs",
   "main": "index.js",
   "files": [
-    "lib/*.js"
+    "lib/**/*"
   ],
   "exports": {
     ".": {
@@ -31,21 +31,25 @@
     "bs58": "^5.0.0",
     "c8": "7",
     "pzp-caps": "^1.0.0",
-    "pzp-db": "^1.0.1",
+    "pzp-db": "^1.0.4",
-    "pzp-dict": "^1.0.0",
+    "pzp-dict": "^1.0.1",
     "pzp-gc": "^1.0.0",
-    "pzp-goals": "^1.0.0",
+    "pzp-goals": "^1.0.1",
     "pzp-keypair": "^1.0.0",
-    "pzp-set": "^1.0.0",
+    "pzp-set": "^1.0.1",
-    "pzp-sync": "^1.0.0",
+    "pzp-sync": "^1.0.4",
     "prettier": "^2.6.2",
     "pretty-quick": "^3.1.3",
     "rimraf": "^4.4.0",
-    "secret-handshake-ext": "~0.0.11",
+    "secret-handshake-ext": "~0.0.12",
     "secret-stack": "~8.1.0",
-    "ssb-box": "^1.0.1"
+    "ssb-box": "^1.0.1",
+    "typescript": "^5.4.5"
   },
   "scripts": {
+    "clean-check": "tsc --build --clean",
+    "prepublishOnly": "npm run clean-check && tsc --build",
+    "postpublish": "npm run clean-check",
     "test": "node --test",
     "format-code": "prettier --write \"(lib|test)/**/*.js\"",
     "format-code-staged": "pretty-quick --staged --pattern \"(lib|test)/**/*.js\"",

test/realtime.test.js

@@ -0,0 +1,171 @@
+const test = require('node:test')
+const assert = require('node:assert')
+const p = require('node:util').promisify
+const { createPeer } = require('./util')
+
+async function flatten(iter) {
+  const ary = []
+  for await (const it of iter) {
+    ary.push(it)
+  }
+  return ary
+}
+
+// copy of test in pzp-sync, but that doesn't test if it works with gc enabled
+test('create 200 messages that manage to replicate with low "newest" goals (using onRecordAdded and with initial message)', async (t) => {
+  const n = 200
+
+  const alice = createPeer({ name: 'alice' })
+  const bob = createPeer({ name: 'bob' })
+
+  await alice.db.loaded()
+  await bob.db.loaded()
+
+  const aliceID = await p(alice.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'alice',
+  })
+  const bobID = await p(bob.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'bob',
+  })
+
+  {
+    const arr = (await flatten(alice.db.msgs()))
+      .filter((msg) => msg.metadata.account === bobID && msg.data)
+      .map((msg) => msg.data.text)
+    assert.deepEqual(arr, [], 'alice has no posts from bob')
+  }
+
+  await p(bob.db.feed.publish)({
+    account: bobID,
+    domain: 'post',
+    data: { text: `${n}` },
+  })
+
+  const confirmed = []
+  // for keeping track of which msgs have arrived
+  for (let i = 0; i < n; i++) {
+    confirmed.push(false)
+  }
+
+  alice.db.onRecordAdded(rec => {
+    if (rec.msg.data?.text) {
+      const num = Number.parseInt(rec.msg.data.text)
+      confirmed[num] = true
+    }
+  })
+
+  await p(alice.set.load)(aliceID)
+  await p(bob.set.load)(bobID)
+
+  assert(await p(alice.set.add)('follows', bobID), 'alice follows bob')
+
+  const goal = 'post@newest-1'
+  await p(alice.conductor.start)(aliceID, [[goal], [goal]], 64_000_000)
+  await p(bob.conductor.start)(bobID, [[goal], [goal]], 64_000_000)
+
+  const remoteAlice = await p(bob.connect)(alice.getAddress())
+  assert('alice and bob connected')
+
+  await p(setTimeout)(1000)
+  assert('sync!')
+
+  const hundred = []
+  for (let i = 0; i < n; i++) {
+      hundred.push(i)
+  }
+  Promise.all(hundred.map(i => p(bob.db.feed.publish)({
+    account: bobID,
+    domain: 'post',
+    data: { text: `${i}` },
+  })))
+  assert(`bob published ${n} posts in parallel`)
+
+  let tries = 30
+  // just waiting for them to arrive
+  do {
+    await p(setTimeout)(100)
+  } while (!confirmed.every(v => v === true) && tries-- > 0)
+
+  assert.equal(confirmed.filter(v => v === true).length, n + 1, `alice has all of bob's posts including the initial one`)
+
+  await p(remoteAlice.close)(true)
+  await p(alice.close)(true)
+  await p(bob.close)(true)
+})
+
+test('create 100 messages in parallel that still manage to sync realtime (without creating an initial post before starting realtime)', async (t) => {
+  const alice = createPeer({ name: 'alice' })
+  const bob = createPeer({ name: 'bob' })
+
+  await alice.db.loaded()
+  await bob.db.loaded()
+
+  const aliceID = await p(alice.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'alice',
+  })
+  const bobID = await p(bob.db.account.create)({
+    subdomain: 'account',
+    _nonce: 'bob',
+  })
+
+  {
+    const arr = (await flatten(alice.db.msgs()))
+      .filter((msg) => msg.metadata.account === bobID && msg.data)
+      .map((msg) => msg.data.text)
+    assert.deepEqual(arr, [], 'alice has no posts from bob')
+  }
+
+  await p(alice.set.load)(aliceID)
+  await p(bob.set.load)(bobID)
+
+  assert(await p(alice.set.add)('follows', bobID), 'alice follows bob')
+
+  const goal = 'post@newest-1'
+  await p(alice.conductor.start)(aliceID, [[goal], [goal]], 64_000_000)
+  await p(bob.conductor.start)(bobID, [[goal], [goal]], 64_000_000)
+
+  const remoteAlice = await p(bob.connect)(alice.getAddress())
+  assert('alice and bob connected')
+
+  await p(setTimeout)(1000)
+  assert('sync!')
+
+  const n = 100
+  const hundred = []
+  for (let i = 0; i < n; i++) {
+      hundred.push(i)
+  }
+  await Promise.all(hundred.map(i => p(bob.db.feed.publish)({
+    account: bobID,
+    domain: 'post',
+    data: { text: `post nr ${i}` },
+  })))
+  assert('bob published 100 posts in parallel')
+
+  const bobMsgs = await flatten(bob.db.msgs())
+  // 1 for creating bob's account, and 1 for the 'post' moot
+  assert.equal(bobMsgs.length, n + 2, "bob has all of his own messages")
+
+  let arr
+  // just waiting for them to arrive
+  for (let i = 0; i < 100; i++) {
+    arr = (await flatten(alice.db.msgs()))
+      // moot doesn't have msg.data
+      .filter((msg) => msg.metadata.account === bobID && msg.data)
+      .filter(msg => msg.metadata.domain === 'post')
+      .map((msg) => msg.data.text)
+    if (arr.length < n) {
+      await p(setTimeout)(100)
+    } else {
+      break
+    }
+  }
+  assert.equal(arr.length, n, `alice has ${arr.length} posts from bob`)
+
+  await p(remoteAlice.close)(true)
+  await p(alice.close)(true)
+  await p(bob.close)(true)
+})