Make all APIs async, internally wait for rescanning (#29)

lib/msg-v4/validation.js

@@ -334,7 +334,7 @@ function validate(msg, tangle, sigkeys, msgID, rootID) {
 
   try {
     if (tangle.type === 'feed' && isMoot(msg)) return // nothing else to check
-  } catch (err) {
+  } catch (/** @type {any} */ err) {
     return err
   }
 

package.json

@@ -29,8 +29,8 @@
   "dependencies": {
     "@alloc/quick-lru": "^5.2.0",
     "atomic-file-rw": "~0.3.0",
-    "blake3": "~2.1.7",
     "b4a": "~1.6.4",
+    "blake3": "~2.1.7",
     "bs58": "~5.0.0",
     "debug": "^4.3.0",
     "is-buffer-zero": "^1.0.0",
@@ -39,15 +39,17 @@
     "multicb": "~1.2.2",
     "mutexify": "~1.4.0",
     "obz": "~1.1.0",
-    "ppppp-keypair": "github:staltz/ppppp-keypair#61ef4420578f450dc2cc7b1efc1c5a691a871c74",
     "polyraf": "^1.1.0",
+    "ppppp-keypair": "github:staltz/ppppp-keypair#61ef4420578f450dc2cc7b1efc1c5a691a871c74",
     "promisify-4loc": "~1.0.0",
     "promisify-tuple": "~1.2.0",
+    "pull-stream": "^3.7.0",
     "push-stream": "~11.2.0",
     "set.prototype.union": "~1.0.2"
   },
   "devDependencies": {
     "@types/b4a": "^1.6.0",
+    "@types/pull-stream": "^3.6.7",
     "c8": "^7.11.0",
     "flumecodec": "~0.0.1",
     "husky": "^4.3.0",
@@ -55,8 +57,8 @@
     "prettier": "^2.6.2",
     "pretty-quick": "^3.1.3",
     "rimraf": "^4.4.0",
-    "secret-stack": "8.0.0",
     "secret-handshake-ext": "0.0.10",
+    "secret-stack": "8.0.0",
     "ssb-box": "^1.0.1",
     "typescript": "^5.1.3"
   },

test/account-add.test.js

@@ -24,7 +24,10 @@ test('account.add()', async (t) => {
       subdomain: 'person',
     })
 
-    assert.equal(peer.db.account.has({ account, keypair: keypair2 }), false)
+    assert.equal(
+      await p(peer.db.account.has)({ account, keypair: keypair2 }),
+      false
+    )
 
     const consent = peer.db.account.consent({ account, keypair: keypair2 })
 
@@ -61,7 +64,10 @@ test('account.add()', async (t) => {
     )
     assert.equal(msg.sigkey, keypair1.public, 'msg.sigkey OLD KEY')
 
-    assert.equal(peer.db.account.has({ account, keypair: keypair2 }), true)
+    assert.equal(
+      await p(peer.db.account.has)({ account, keypair: keypair2 }),
+      true
+    )
 
     await p(peer.close)()
   })
@@ -79,7 +85,7 @@ test('account.add()', async (t) => {
       keypair: keypair1,
       subdomain: 'account',
     })
-    const msg1 = peer1.db.get(id)
+    const msg1 = await p(peer1.db.get)(id)
 
     const { msg: msg2 } = await p(peer1.db.account.add)({
       account: id,
@@ -88,7 +94,10 @@ test('account.add()', async (t) => {
     })
     assert.equal(msg2.data.key.bytes, keypair2.public)
 
-    assert.equal(peer1.db.account.has({ account: id, keypair: keypair2 }), true)
+    assert.equal(
+      await p(peer1.db.account.has)({ account: id, keypair: keypair2 }),
+      true
+    )
 
     await p(peer1.close)()
     rimraf.sync(DIR)
@@ -151,7 +160,7 @@ test('account.add()', async (t) => {
       keypair: keypair1,
       subdomain: 'person',
     })
-    const accountMsg0 = peer.db.get(account)
+    const accountMsg0 = await p(peer.db.get)(account)
 
     // Consent is implicitly created because keypair2 has .private
     const accountRec1 = await p(peer.db.account.add)({
@@ -166,10 +175,13 @@ test('account.add()', async (t) => {
       keypair: keypair2,
     })
     assert.equal(postRec.msg.data.text, 'hello', 'post text correct')
-    const mootRec = peer.db.feed.findMoot(account, 'post')
+    const mootRec = await p(peer.db.feed.findMoot)(account, 'post')
     assert.ok(mootRec, 'posts moot exists')
 
-    const recs = [...peer.db.records()]
+    const recs = []
+    for await (rec of peer.db.records()) {
+      recs.push(rec)
+    }
     assert.equal(recs.length, 4, '4 records')
     const [_accountRec0, _accountRec1, postsRoot, _post] = recs
     assert.deepEqual(_accountRec0.msg, accountMsg0, 'accountMsg0')
@@ -224,7 +236,7 @@ test('account.add()', async (t) => {
         keypair: keypair1,
         subdomain: 'person',
       })
-      const accountMsg0 = peer.db.get(account)
+      const accountMsg0 = await p(peer.db.get)(account)
 
       const consent = peer.db.account.consent({ account, keypair: keypair2 })
 
@@ -241,7 +253,7 @@ test('account.add()', async (t) => {
         data: { text: 'potato' },
         keypair: keypair2,
       })
-      const postMootRec = peer.db.feed.findMoot(account, 'post')
+      const postMootRec = await p(peer.db.feed.findMoot)(account, 'post')
 
       const delRec = await p(peer.db.account.del)({
         account,

test/account-create.test.js

@@ -21,7 +21,7 @@ test('account.create() ', async (t) => {
       _nonce: 'MYNONCE',
     })
     assert.ok(account, 'accountRec0 exists')
-    const msg = peer.db.get(account)
+    const msg = await p(peer.db.get)(account)
     assert.deepEqual(
       msg.data,
       {
@@ -60,7 +60,7 @@ test('account.create() ', async (t) => {
       subdomain: 'person',
     })
     assert.ok(account, 'account created')
-    const msg = peer.db.get(account)
+    const msg = await p(peer.db.get)(account)
     assert.equal(msg.data.key.bytes, keypair.public, 'msg.data')
     assert.equal(msg.metadata.account, 'self', 'msg.metadata.account')
     assert.equal(msg.metadata.accountTips, null, 'msg.metadata.accountTips')
@@ -129,7 +129,7 @@ test('account.create() ', async (t) => {
       subdomain,
     })
     assert.ok(account, 'account created')
-    const msg = peer.db.get(account)
+    const msg = await p(peer.db.get)(account)
     assert.equal(msg.data.key.bytes, keypair.public, 'msg.data')
     assert.equal(msg.metadata.account, 'self', 'msg.metadata.account')
     assert.equal(msg.metadata.accountTips, null, 'msg.metadata.accountTips')

test/add.test.js

@@ -103,7 +103,7 @@ test('add()', async (t) => {
     {
       const ids = []
       const texts = []
-      for (const rec of peer.db.records()) {
+      for await (const rec of peer.db.records()) {
         if (rec.msg.metadata.domain === 'something') {
           ids.push(rec.id)
           texts.push(rec.msg.data?.text)
@@ -122,7 +122,7 @@ test('add()', async (t) => {
     {
       const ids = []
       const texts = []
-      for (const rec of peer.db.records()) {
+      for await (const rec of peer.db.records()) {
         if (rec.msg.metadata.domain === 'something') {
           ids.push(rec.id)
           texts.push(rec.msg.data?.text)

test/del.test.js

@@ -36,7 +36,7 @@ test('del()', async (t) => {
 
   {
     const texts = []
-    for (const msg of peer.db.msgs()) {
+    for await (const msg of peer.db.msgs()) {
       if (msg.data && msg.metadata.account?.length > 4) {
         texts.push(msg.data.text)
       }
@@ -60,7 +60,7 @@ test('del()', async (t) => {
 
   {
     const texts = []
-    for (const msg of peer.db.msgs()) {
+    for await (const msg of peer.db.msgs()) {
       if (msg.data && msg.metadata.account?.length > 4) {
         texts.push(msg.data.text)
       }
@@ -76,7 +76,7 @@ test('del()', async (t) => {
 
   {
     const texts = []
-    for (const msg of peer.db.msgs()) {
+    for await (const msg of peer.db.msgs()) {
       if (msg.data && msg.metadata.account?.length > 4) {
         texts.push(msg.data.text)
       }

test/erase.test.js

@@ -36,7 +36,7 @@ test('erase()', async (t) => {
   const SAVED_UPON_ERASE = '{"text":"m*"}'.length - 'null'.length
 
   const before = []
-  for (const msg of peer.db.msgs()) {
+  for await (const msg of peer.db.msgs()) {
     if (msg.data && msg.metadata.account?.length > 4) {
       before.push(msg.data.text)
     }
@@ -59,7 +59,7 @@ test('erase()', async (t) => {
   await p(peer.db.erase)(msgIDs[2])
 
   const after = []
-  for (const msg of peer.db.msgs()) {
+  for await (const msg of peer.db.msgs()) {
     if (msg.data && msg.metadata.account?.length > 4) {
       after.push(msg.data.text)
     }
@@ -68,7 +68,7 @@ test('erase()', async (t) => {
   assert.deepEqual(after, ['m0', 'm1', 'm3', 'm4'], '4 msgs after the erase')
 
   const after2 = []
-  for (const msg of peer.db.msgs()) {
+  for await (const msg of peer.db.msgs()) {
     for (const tangleID in msg.metadata.tangles) {
       after2.push(msg.metadata.tangles[tangleID].depth)
     }

test/feed-find-moot.test.js

@@ -23,7 +23,7 @@ test('feed.findMoot()', async (t) => {
 
   await p(peer.db.add)(moot, mootID)
 
-  const mootRec = peer.db.feed.findMoot(id, 'post')
+  const mootRec = await p(peer.db.feed.findMoot)(id, 'post')
   assert.equal(mootRec.id, mootID, 'feed.findMoot() returns moot ID')
 
   await p(peer.close)(true)

test/feed-publish.test.js

@@ -127,7 +127,7 @@ test('feed.publish()', async (t) => {
     assert.equal(typeof recEncrypted.msg.data, 'string')
     assert.ok(recEncrypted.msg.data.endsWith('.box'), '.box')
 
-    const msgDecrypted = peer.db.get(recEncrypted.id)
+    const msgDecrypted = await p(peer.db.get)(recEncrypted.id)
     assert.equal(msgDecrypted.data.text, 'I am chewing food')
   })
 

test/get.test.js

@@ -28,7 +28,7 @@ test('get()', async (t) => {
   })
   const msgID1 = MsgV4.getMsgID(rec1.msg)
 
-  const msg = peer.db.get(msgID1)
+  const msg = await p(peer.db.get)(msgID1)
   assert.ok(msg, 'msg exists')
   assert.equal(msg.data.text, 'I am 1st post')
 

test/getTangle.test.js

@@ -103,12 +103,12 @@ test('getTangle()', async (t) => {
     reply3LoText = reply3B.localeCompare(reply3C) < 0 ? 'reply 3B' : 'reply 3C'
     reply3HiText = reply3B.localeCompare(reply3C) < 0 ? 'reply 3C' : 'reply 3B'
 
-    tangle = peer.db.getTangle(rootPost)
+    tangle = await p(peer.db.getTangle)(rootPost)
   }
 
-  await t.test('getTangle unknown ID returns null', (t) => {
+  await t.test('getTangle unknown ID returns null', async (t) => {
     assert.equal(
-      peer.db.getTangle('Lq6xwbdvGVmSsY3oYRugpZ3DY8chX9SLhRhjJKyZHQn'),
+      await p(peer.db.getTangle)('Lq6xwbdvGVmSsY3oYRugpZ3DY8chX9SLhRhjJKyZHQn'),
       null
     )
   })
@@ -280,9 +280,9 @@ test('getTangle()', async (t) => {
     assert.deepEqual(actual4, expected4)
   })
 
-  await t.test('Tangle.slice', (t) => {
+  await t.test('Tangle.slice', async (t) => {
     {
-      const msgs = tangle.slice()
+      const msgs = await tangle.slice()
       const texts = msgs.map((msg) => msg.data?.text)
       assert.deepEqual(texts, [
         'root',
@@ -295,13 +295,13 @@ test('getTangle()', async (t) => {
     }
 
     {
-      const msgs = tangle.slice([], [reply2])
+      const msgs = await tangle.slice([], [reply2])
       const texts = msgs.map((msg) => msg.data?.text)
       assert.deepEqual(texts, ['root', reply1LoText, reply1HiText, 'reply 2'])
     }
 
     {
-      const msgs = tangle.slice([reply2], [])
+      const msgs = await tangle.slice([reply2], [])
       const texts = msgs.map((msg) => msg.data?.text)
       assert.deepEqual(texts, [
         undefined, // root
@@ -313,7 +313,7 @@ test('getTangle()', async (t) => {
     }
 
     {
-      const msgs = tangle.slice([reply2], [reply2])
+      const msgs = await tangle.slice([reply2], [reply2])
       const texts = msgs.map((msg) => msg.data?.text)
       assert.deepEqual(texts, [
         undefined, // root
@@ -323,7 +323,7 @@ test('getTangle()', async (t) => {
     }
 
     {
-      const msgs = tangle.slice([reply2], [reply2, reply3Lo])
+      const msgs = await tangle.slice([reply2], [reply2, reply3Lo])
       const texts = msgs.map((msg) => msg.data?.text)
       assert.deepEqual(texts, [
         undefined, // root
@@ -343,7 +343,7 @@ test('getTangle()', async (t) => {
       await p(peer.db.erase)(msgID)
     }
 
-    const tangle2 = peer.db.getTangle(rootPost)
+    const tangle2 = await p(peer.db.getTangle)(rootPost)
     const sorted = tangle2.topoSort()
 
     assert.deepEqual(sorted, [rootPost, reply3Lo, reply3Hi])

test/ghosts.tests.js

@@ -6,6 +6,7 @@ const p = require('node:util').promisify
 const rimraf = require('rimraf')
 const Keypair = require('ppppp-keypair')
 const { createPeer } = require('./util')
+const MsgV4 = require('../lib/msg-v4')
 
 const DIR = path.join(os.tmpdir(), 'ppppp-db-ghosts')
 rimraf.sync(DIR)
@@ -28,7 +29,7 @@ test('ghosts.add, ghosts.get, ghosts.getMinDepth', async (t) => {
     })
     msgIDs.push(rec.id)
   }
-  const tangleID = peer.db.feed.findMoot(account, 'post')?.id
+  const tangleID = (await p(peer.db.feed.findMoot)(account, 'post'))?.id
 
   const ghosts0 = peer.db.ghosts.get(tangleID)
   assert.deepEqual(ghosts0, [], 'no ghosts so far')
@@ -71,7 +72,8 @@ test('ghosts.add queues very-concurrent calls', async (t) => {
     })
     msgIDs.push(rec.id)
   }
-  const tangleID = peer.db.feed.findMoot(account, 'post')?.id
+  const moot = MsgV4.createMoot(account, 'post', keypair)
+  const tangleID = MsgV4.getMsgID(moot)
 
   const ghosts0 = peer.db.ghosts.get(tangleID)
   assert.deepEqual(ghosts0, [], 'no ghosts so far')

test/msgs-iterator.test.js

@@ -31,7 +31,7 @@ test('msgs() iterator', async (t) => {
 
   const posts = []
   const abouts = []
-  for (const msg of peer.db.msgs()) {
+  for await (const msg of peer.db.msgs()) {
     if (!msg.data) continue
     if (msg.metadata.domain === 'post') posts.push(msg.data.text)
     else if (msg.metadata.domain === 'about') abouts.push(msg.data.name)

test/re-open.test.js

@@ -41,7 +41,7 @@ test('publish some msgs, close, re-open', async (t) => {
   await peer2.db.loaded()
 
   const texts = []
-  for (const msg of peer2.db.msgs()) {
+  for await (const msg of peer2.db.msgs()) {
     if (!msg.data || !(msg.metadata.account?.length > 4)) continue
     texts.push(msg.data.text)
   }

test/records-iterator.test.js

@@ -29,7 +29,7 @@ test('records() iterator', async (t) => {
   }
 
   let count = 0
-  for (const rec of peer.db.records()) {
+  for await (const rec of peer.db.records()) {
     if (!rec.msg.data) continue
     if (rec.msg.metadata.account === 'self') continue
     assert.ok(rec.received, 'received')

test/sigkeys.test.js

@@ -31,7 +31,7 @@ test('sigkeys', async (t) => {
         keypair: keypair1,
         subdomain: 'person',
       })
-      const accountMsg0 = peer.db.get(account)
+      const accountMsg0 = await p(peer.db.get)(account)
 
       const consent = peer.db.account.consent({ account, keypair: keypair2 })
 
@@ -50,7 +50,7 @@ test('sigkeys', async (t) => {
       })
 
       const postMootId = peer.db.feed.getID(account, 'post')
-      const postMootMsg = peer.db.get(postMootId)
+      const postMootMsg = await p(peer.db.get)(postMootId)
 
       const tangle = new MsgV4.Tangle(postMootId)
       tangle.add(postMootId, postMootMsg)

tsconfig.json

@@ -1,17 +1,31 @@
 {
-  "include": ["declarations", "lib/**/*.js"],
-  "exclude": ["coverage/", "node_modules/", "test/"],
+  "include": [
+    "declarations",
+    "lib/**/*.js"
+  ],
+  "exclude": [
+    "coverage/",
+    "node_modules/",
+    "test/"
+  ],
   "compilerOptions": {
     "checkJs": true,
     "declaration": true,
     "emitDeclarationOnly": true,
     "exactOptionalPropertyTypes": true,
     "forceConsistentCasingInFileNames": true,
-    "lib": ["es2022", "dom"],
+    "noImplicitReturns": true,
+    "lib": [
+      "es2022",
+      "dom"
+    ],
     "module": "node16",
     "skipLibCheck": true,
     "strict": true,
     "target": "es2022",
-    "typeRoots": ["node_modules/@types", "declarations"]
+    "typeRoots": [
+      "node_modules/@types",
+      "declarations"
+    ]
   }
 }