1.0.4

Trying to fix https://codeberg.org/pzp/pzp-sync/issues/10

This doesn't solve the problem of missing rootMsg but maybe allows sync to fail more gracefully

Reviewed-on: https://codeberg.org/pzp/pzp-db/pulls/8
Co-authored-by: Jacob Karlsson <jacob.karlsson95@gmail.com>
Co-committed-by: Jacob Karlsson <jacob.karlsson95@gmail.com>

lib/db-tangle.js

@@ -0,0 +1,158 @@
+const pull = require('pull-stream')
+const p = require('node:util').promisify
+const MsgV4 = require('./msg-v4')
+
+/**
+ * @typedef {string} MsgID
+ * @typedef {import('./msg-v4').Msg} Msg
+ */
+
+/**
+ * @typedef {{
+ *   id?: never;
+ *   msg?: never;
+ *   received?: never;
+ * }} RecDeleted
+ *
+ * @typedef {{
+ *   id: MsgID;
+ *   msg: Msg;
+ *   received: number;
+ * }} RecPresent
+ *
+ * @typedef {RecPresent | RecDeleted} Rec
+ */
+
+/**
+ * @template T
+ * @typedef {[T] extends [void] ?
+ *   (...args: [Error] | []) => void :
+ *   (...args: [Error] | [null, T]) => void
+ * } CB
+ */
+
+class DBTangle extends MsgV4.Tangle {
+  /** @type {(msgID: MsgID, cb: CB<Msg>) => void} */
+  #getMsg
+
+  /**
+   * @param {MsgID} rootID
+   * @param {(msgID: MsgID, cb: CB<Msg>) => void} getMsg
+   */
+  constructor(rootID, getMsg) {
+    super(rootID)
+    this.#getMsg = getMsg
+  }
+
+  /**
+   * @param {MsgID} rootID
+   * @param {AsyncIterable<Rec>} recordsIter
+   * @param {(msgID: MsgID, cb: any) => void} getMsg
+   * @return {Promise<DBTangle>}
+   */
+  static async init(rootID, recordsIter, getMsg) {
+    const dbtangle = new DBTangle(rootID, getMsg)
+    for await (const rec of recordsIter) {
+      if (!rec.msg) continue
+      dbtangle.add(rec.id, rec.msg)
+    }
+    return dbtangle
+  }
+
+  /**
+   * Given a set of msgs (`msgIDs`) in this tangle, find all "deletable" and
+   * "erasable" msgs that precede that set.
+   *
+   * *Deletables* are msgs that precede `msgsIDs` but are not important in any
+   * validation path toward the root, and thus can be deleted.
+   *
+   * *Erasables* are msgs that precede `msgsIDs` and can be erased without
+   * losing a validation path toward the root.
+   * @param {Array<MsgID>} msgIDs
+   * @returns {{ deletables: Set<MsgID>, erasables: Set<MsgID> } | null}
+   */
+  getDeletablesAndErasables(...msgIDs) {
+    // Determine erasables
+    const erasables = new Set()
+    const minimum = this.getMinimumAmong(msgIDs)
+    for (const msgID of minimum) {
+      const trail = this.shortestPathToRoot(msgID)
+      if (!trail) return null
+      for (const id of trail) {
+        erasables.add(id)
+      }
+    }
+
+    // Determine deletables
+    const deletables = new Set()
+    const sorted = this.topoSort()
+    for (const msgID of sorted) {
+      if (erasables.has(msgID)) continue
+      if (minimum.some((min) => this.precedes(msgID, min))) {
+        deletables.add(msgID)
+      }
+    }
+
+    return { deletables, erasables }
+  }
+
+  /**
+   * @param {Array<string>=} minSet
+   * @param {Array<string>=} maxSet
+   * @param {CB<Array<Msg>>=} cb
+   * @return {Promise<Array<Msg>>|void}
+   */
+  slice(minSet = [], maxSet = [], cb) {
+    // @ts-ignore
+    if (cb === undefined) return p(this.slice).bind(this)(minSet, maxSet)
+
+    const minSetGood = minSet.filter((msgID) => this.has(msgID))
+    const maxSetGood = maxSet.filter((msgID) => this.has(msgID))
+    const minSetTight = this.getMinimumAmong(minSetGood)
+
+    const trail = new Set()
+    for (const msgID of minSetTight) {
+      const path = this.shortestPathToRoot(msgID)
+      if (!path) return cb(Error("Couldn't get shortest path to root when slicing dbtangle"))
+      for (const msgID of path) {
+        trail.add(msgID)
+      }
+    }
+
+    const msgs = /**@type {Array<Msg>}*/ ([])
+
+    pull(
+      pull.values(this.topoSort()),
+      pull.asyncMap((msgID, cb) => {
+        this.#getMsg(msgID, (err, msg) => {
+          if (err) return cb(err)
+          cb(null, { id: msgID, msg })
+        })
+      }),
+      pull.drain(
+        (rec) => {
+          if (trail.has(rec.id)) {
+            if (rec.msg) msgs.push({ ...rec.msg, data: null })
+          }
+          const isMin = minSetGood.includes(rec.id)
+          const isMax = maxSetGood.includes(rec.id)
+          const isBeforeMin = minSetGood.some((min) =>
+            this.precedes(rec.id, min)
+          )
+          const isAfterMax = maxSetGood.some((max) =>
+            this.precedes(max, rec.id)
+          )
+          if (!isMin && isBeforeMin) return
+          if (!isMax && isAfterMax) return
+          if (rec.msg) msgs.push(rec.msg)
+        },
+        (err) => {
+          if (err) return cb(Error('DBTangle.slice() failed', { cause: err }))
+          return cb(null, msgs)
+        }
+      )
+    )
+  }
+}
+
+module.exports = DBTangle

lib/index.js

@@ -8,6 +8,7 @@ const pull = require('pull-stream')
 const p = require('node:util').promisify
 const Log = require('./log')
 const MsgV4 = require('./msg-v4')
+const DBTangle = require('./db-tangle')
 const {
   SIGNATURE_TAG_ACCOUNT_ADD,
   ACCOUNT_SELF,
@@ -72,7 +73,7 @@ const { decrypt } = require('./encryption')
 
 /**
  * @template T
- * @typedef {T extends void ?
+ * @typedef {[T] extends [void] ?
  *   (...args: [Error] | []) => void :
  *   (...args: [Error] | [null, T]) => void
  * } CB
@@ -93,128 +94,6 @@ function assertValidConfig(config) {
   }
 }
 
-class DBTangle extends MsgV4.Tangle {
-  /** @type {(msgID: MsgID, cb: CB<Msg>) => void} */
-  #getMsg
-
-  /**
-   * @param {MsgID} rootID
-   * @param {(msgID: MsgID, cb: CB<Msg>) => void} getMsg
-   */
-  constructor(rootID, getMsg) {
-    super(rootID)
-    this.#getMsg = getMsg
-  }
-
-  /**
-   * @param {MsgID} rootID
-   * @param {AsyncIterable<Rec>} recordsIter
-   * @param {(msgID: MsgID, cb: any) => void} getMsg
-   * @return {Promise<DBTangle>}
-   */
-  static async init(rootID, recordsIter, getMsg) {
-    const dbtangle = new DBTangle(rootID, getMsg)
-    for await (const rec of recordsIter) {
-      if (!rec.msg) continue
-      dbtangle.add(rec.id, rec.msg)
-    }
-    return dbtangle
-  }
-
-  /**
-   * Given a set of msgs (`msgIDs`) in this tangle, find all "deletable" and
-   * "erasable" msgs that precede that set.
-   *
-   * *Deletables* are msgs that precede `msgsIDs` but are not important in any
-   * validation path toward the root, and thus can be deleted.
-   *
-   * *Erasables* are msgs that precede `msgsIDs` and can be erased without
-   * losing a validation path toward the root.
-   * @param {Array<MsgID>} msgIDs
-   * @returns {{ deletables: Set<MsgID>, erasables: Set<MsgID> }}
-   */
-  getDeletablesAndErasables(...msgIDs) {
-    // Determine erasables
-    const erasables = new Set()
-    const minimum = this.getMinimumAmong(msgIDs)
-    for (const msgID of minimum) {
-      const trail = this.shortestPathToRoot(msgID)
-      for (const id of trail) {
-        erasables.add(id)
-      }
-    }
-
-    // Determine deletables
-    const deletables = new Set()
-    const sorted = this.topoSort()
-    for (const msgID of sorted) {
-      if (erasables.has(msgID)) continue
-      if (minimum.some((min) => this.precedes(msgID, min))) {
-        deletables.add(msgID)
-      }
-    }
-
-    return { deletables, erasables }
-  }
-
-  /**
-   * @param {Array<string>=} minSet
-   * @param {Array<string>=} maxSet
-   * @param {CB<Array<Msg>>=} cb
-   * @return {Promise<Array<Msg>>|void}
-   */
-  slice(minSet = [], maxSet = [], cb) {
-    // @ts-ignore
-    if (cb === undefined) return p(this.slice).bind(this)(minSet, maxSet)
-
-    const minSetGood = minSet.filter((msgID) => this.has(msgID))
-    const maxSetGood = maxSet.filter((msgID) => this.has(msgID))
-    const minSetTight = this.getMinimumAmong(minSetGood)
-
-    const trail = new Set()
-    for (const msgID of minSetTight) {
-      const path = this.shortestPathToRoot(msgID)
-      for (const msgID of path) {
-        trail.add(msgID)
-      }
-    }
-
-    const msgs = /**@type {Array<Msg>}*/ ([])
-
-    pull(
-      pull.values(this.topoSort()),
-      pull.asyncMap((msgID, cb) => {
-        this.#getMsg(msgID, (err, msg) => {
-          if (err) return cb(err)
-          cb(null, { id: msgID, msg })
-        })
-      }),
-      pull.drain(
-        (rec) => {
-          if (trail.has(rec.id)) {
-            if (rec.msg) msgs.push({ ...rec.msg, data: null })
-          }
-          const isMin = minSetGood.includes(rec.id)
-          const isMax = maxSetGood.includes(rec.id)
-          const isBeforeMin = minSetGood.some((min) =>
-            this.precedes(rec.id, min)
-          )
-          const isAfterMax = maxSetGood.some((max) =>
-            this.precedes(max, rec.id)
-          )
-          if (!isMin && isBeforeMin) return
-          if (!isMax && isAfterMax) return
-          if (rec.msg) msgs.push(rec.msg)
-        },
-        (err) => {
-          if (err) return cb(Error('DBTangle.slice() failed', { cause: err }))
-          return cb(null, msgs)
-        }
-      )
-    )
-  }
-}
-
 /**
  * @param {Peer} peer
  * @param {Config} config
@@ -421,7 +300,7 @@ function initDB(peer, config) {
 
   /**
    * @param {Pick<RecPresent, 'id' | 'msg'>} rec
-   * @param {(err: Error | null, tangle: DBTangle | null) => void} cb
+   * @param {CB<DBTangle | null>} cb
    */
   function getAccountTangle(rec, cb) {
     const accountID = getAccountID(rec)
@@ -432,8 +311,7 @@ function initDB(peer, config) {
         }
         if (!accountTangle.has(accountID)) {
           return cb(
-            Error(`Account tangle "${accountID}" is locally unknown`),
+            Error(`Account tangle "${accountID}" is locally unknown`)
-            null
           )
         }
         return cb(null, accountTangle)
@@ -483,7 +361,7 @@ function initDB(peer, config) {
   }
 
   /**
-   * @param {CB<void>} cb
+   * @param {CB<void> | void} cb
    */
   function loaded(cb) {
     if (cb === void 0) return promisify(loaded)()
@@ -499,7 +377,7 @@ function initDB(peer, config) {
    *
    * @param {Pick<RecPresent, 'id' | 'msg'>} rec
    * @param {MsgID} tangleID
-   * @param {(err: Error | null, val: null) => void} cb
+   * @param {CB<void>} cb
    */
   function verifyRec(rec, tangleID, cb) {
     let err
@@ -515,34 +393,35 @@ function initDB(peer, config) {
         if (
           (err = MsgV4.validate(rec.msg, tangle, sigkeys, rec.id, tangleID))
         ) {
-          return cb(Error('Invalid msg', { cause: err }), null)
+          return cb(Error('Invalid msg', { cause: err }))
         }
-        return cb(null, null)
+        return cb()
       }
 
       // Identify the account and its sigkeys:
       getAccountTangle(rec, (err, accountTangle) => {
         // prettier-ignore
-        if (err) return cb(Error('Unknown account tangle owning this msg', { cause: err }), null)
+        if (err) return cb(Error('Unknown account tangle owning this msg', { cause: err }))
 
         getSigkeysInAccount(
           accountTangle,
           rec.msg.metadata.accountTips,
           (err, sigkeys) => {
-            if (err) return cb(err, null)
+            if (err) return cb(err)
-
+            // TODO: how is sigkeys able to be undefined here? typechecker why are you weird?
+            if (!sigkeys) return cb(Error("Sigkeys missing somehow"))
             // Don't accept ghosts to come back, unless they are trail msgs
             if (!!rec.msg.data && ghosts.read(tangleID).has(rec.id)) {
-              return cb(Error('Refusing a ghost msg to come back'), null)
+              return cb(Error('Refusing a ghost msg to come back'))
             }
 
-            if (
+            
-              (err = MsgV4.validate(rec.msg, tangle, sigkeys, rec.id, tangleID))
+            const valErr = MsgV4.validate(rec.msg, tangle, sigkeys, rec.id, tangleID)
-            ) {
+            if (valErr) {
-              return cb(Error('Invalid msg', { cause: err }), null)
+              return cb(Error('Invalid msg', { cause: valErr }))
             }
 
-            /** @param {(err: Error | null, val: null) => void} cb */
+            /** @param {CB<void>} cb */
             function verifyInner(cb) {
               // Unwrap encrypted inner msg and verify it too
               if (typeof rec.msg.data === 'string') {
@@ -554,15 +433,15 @@ function initDB(peer, config) {
 
                   verifyRec(innerRec, innerMsgID, (err) => {
                     // prettier-ignore
-                    if (err) return cb(Error('Failed to verify inner msg', { cause: err }), null)
+                    if (err) return cb(Error('Failed to verify inner msg', { cause: err }))
 
-                    return cb(null, null)
+                    return cb()
                   })
                 } else {
-                  return cb(null, null)
+                  return cb()
                 }
               } else {
-                return cb(null, null)
+                return cb()
               }
             }
 
@@ -571,7 +450,7 @@ function initDB(peer, config) {
               const validAccountTangle = /** @type {Tangle} */ (accountTangle)
               validateAccountMsg(rec.msg, validAccountTangle, (err) => {
                 if (err)
-                  return cb(Error('Invalid account msg', { cause: err }), null)
+                  return cb(Error('Invalid account msg', { cause: err }))
                 return verifyInner(cb)
               })
             } else {
@@ -677,7 +556,7 @@ function initDB(peer, config) {
   /**
    * @param {Msg} msg
    * @param {Tangle} accountTangle
-   * @param {(err: Error | null, val: null) => void} cb
+   * @param {CB<void>} cb
    */
   function validateAccountMsg(msg, accountTangle, cb) {
     if (!MsgV4.isRoot(msg)) {
@@ -690,21 +569,21 @@ function initDB(peer, config) {
           public: msg.sigkey,
         }
         getAccountPowers(accountTangle, keypair, (err, powers) => {
-          if (err) return cb(err, null)
+          if (err) return cb(err)
 
           if (!powers.has('add')) {
             // prettier-ignore
-            return cb(Error(`invalid account msg: sigkey "${msg.sigkey}" does not have "add" power`), null)
+            return cb(Error(`invalid account msg: sigkey "${msg.sigkey}" does not have "add" power`))
           }
 
-          return cb(null, null)
+          return cb()
         })
       } else {
-        return cb(null, null)
+        return cb()
       }
       // TODO validate 'del'
     } else {
-      return cb(null, null)
+      return cb()
     }
   }
 
@@ -800,7 +679,7 @@ function initDB(peer, config) {
    *   keypair?: KeypairPublicSlice;
    *   account: string;
    * }} opts
-   * @param {(err: Error | null, has: boolean | null) => void} cb
+   * @param {CB<boolean>} cb
    */
   function accountHas(opts, cb) {
     const keypair = opts?.keypair ?? config.global.keypair
@@ -811,7 +690,7 @@ function initDB(peer, config) {
         pull.asyncMap((msgID, cb) => {
           get(msgID, (err, msg) => {
             // prettier-ignore
-            if (err) return cb(Error("db.account.has() failed to get() account tangle message", { cause: err }), null)
+            if (err) return cb(Error("db.account.has() failed to get() account tangle message", { cause: err }))
 
             if (!msg?.data) return cb(null, false)
             /** @type {AccountData} */
@@ -826,7 +705,7 @@ function initDB(peer, config) {
         }),
         pull.collect((err, results) => {
           // prettier-ignore
-          if (err) return cb(Error('db.account.has() failed to calculate', { cause: err }), null)
+          if (err) return cb(Error('db.account.has() failed to calculate', { cause: err }))
 
           return cb(
             null,
@@ -897,7 +776,6 @@ function initDB(peer, config) {
     })
   }
 
-  //* @param {(err: Error | null, val: Set<any> | null) => void} cb
   /**
    * @param {Tangle} accountTangle
    * @param {KeypairPublicSlice} keypair
@@ -1286,7 +1164,7 @@ function initDB(peer, config) {
   /**
    * @param {string} accountId
    * @param {string} domain
-   * @param {(err: Error | null, rec: RecPresent | null) => void} cb
+   * @param {CB<RecPresent | null>} cb
    */
   function findMoot(accountId, domain, cb) {
     const findAccount = MsgV4.stripAccount(accountId)
@@ -1303,7 +1181,7 @@ function initDB(peer, config) {
 
   /**
    * @param {MsgID} msgID
-   * @param {(err: Error | null, rec: RecPresent | null) => void} cb
+   * @param {CB<RecPresent | null>} cb
    */
   function getRecord(msgID, cb) {
     // TODO: improve performance of this when getting many messages, the arg
@@ -1322,7 +1200,7 @@ function initDB(peer, config) {
 
   /**
    * @param {MsgID} msgID
-   * @param {(err: Error | null, msg?: Msg) => void} cb
+   * @param {CB<Msg | undefined>} cb
    */
   function get(msgID, cb) {
     getRecord(msgID, (err, rec) => {
@@ -1466,7 +1344,7 @@ function initDB(peer, config) {
 
   /**
    * @param {MsgID} tangleID
-   * @param {(err: Error | null, tangle: DBTangle | null) => void} cb
+   * @param {CB<DBTangle | null>} cb
    */
   function getTangle(tangleID, cb) {
     DBTangle.init(tangleID, records(), get).then((tangle) => {

lib/msg-v4/tangle.js

@@ -222,11 +222,12 @@ class Tangle {
   }
 
   /**
-   * @returns {'feed' | 'account' | 'weave'}
+   * @returns {'feed' | 'account' | 'weave' | null}
    */
   get type() {
     if (!this.#rootMsg) {
-      throw new Error(`Tangle "${this.#rootID}" is missing root message`)
+      console.trace(new Error(`Tangle "${this.#rootID}" is missing root message`))
+      return null
     }
     if (this.#isFeed()) return 'feed'
     if (this.#rootMsg.metadata.account === 'self') return 'account'
@@ -235,7 +236,8 @@ class Tangle {
 
   get root() {
     if (!this.#rootMsg) {
-      throw new Error(`Tangle "${this.#rootID}" is missing root message`)
+      console.trace(new Error(`Tangle "${this.#rootID}" is missing root message`))
+      return null
     }
     return this.#rootMsg
   }
@@ -256,7 +258,8 @@ class Tangle {
       if (!prev) break
       if (prev === lastPrev) {
         // prettier-ignore
-        throw new Error(`Tangle "${this.#rootID}" has a cycle or lacking a trail to root`)
+        console.trace(new Error(`Tangle "${this.#rootID}" has a cycle or lacking a trail to root`))
+        return null
       } else {
         lastPrev = prev
       }

lib/msg-v4/validation.js

@@ -55,6 +55,7 @@ function validateShape(msg) {
   if (typeof msg.sig !== 'string') {
     return 'invalid msg: must have sig\n' + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -77,6 +78,7 @@ function validateSigkey(msg) {
     // prettier-ignore
     return `invalid msg: sigkey "${sigkey}" should have been a base58 string\n` + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -105,7 +107,10 @@ function validateSigkeyAndAccount(msg, tangle, sigkeys) {
       // prettier-ignore
       return `invalid msg: accountTips "${msg.metadata.accountTips}" should have been null in an account tangle\n` + JSON.stringify(msg)
     }
+  } else if (tangle.type === null) {
+    return "Cannot validate tangle of unknown type"
   }
+  return undefined
 }
 
 /**
@@ -122,6 +127,7 @@ function validateMsgID(str) {
   } catch (err) {
     return `invalid msgID "${str}": should have been a base58 string`
   }
+  return undefined
 }
 
 /**
@@ -158,6 +164,7 @@ function validateSignature(msg) {
   if (!verified) {
     return 'invalid msg: sig is invalid\n' + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -194,6 +201,8 @@ function validateTangle(msg, tangle, tangleID) {
       // prettier-ignore
       return `invalid msg: account "${msg.metadata.account}" should have been feed account "${account}"\n` + JSON.stringify(msg)
     }
+  } else if (tangle.type === null) {
+    return "Unknown tangle type"
   }
   let lastPrev = null
   let minDiff = Infinity
@@ -241,6 +250,7 @@ function validateTangle(msg, tangle, tangleID) {
     // prettier-ignore
     return `invalid msg: depth must be the largest prev depth plus one\n` + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -257,6 +267,7 @@ function validateTangleRoot(msg, msgID, tangleID) {
     // prettier-ignore
     return `invalid msg: tangle root "${tangleID}" must not have self tangle data\n` + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -277,6 +288,7 @@ function validateDomain(domain) {
     // prettier-ignore
     return `invalid domain: "${domain}" contains characters other than a-z, A-Z, 0-9, or _`
   }
+  return undefined
 }
 
 /**
@@ -295,6 +307,7 @@ function validateData(msg) {
     // prettier-ignore
     return `invalid msg: data "${data}" must be an object or a string` + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -317,6 +330,7 @@ function validateDataSizeHash(msg) {
     // prettier-ignore
     return `invalid msg: metadata.dataSize ${actualSize} should have been "${expectedSize}"\n` + JSON.stringify(msg)
   }
+  return undefined
 }
 
 /**
@@ -332,11 +346,8 @@ function validate(msg, tangle, sigkeys, msgID, rootID) {
   if ((err = validateSigkey(msg))) return err
   if ((err = validateData(msg))) return err
 
-  try {
+  if (tangle.type === 'feed' && isMoot(msg)) return // nothing else to check
-    if (tangle.type === 'feed' && isMoot(msg)) return // nothing else to check
+  if (tangle.type === null) return "Missing tangle type when validating msg"
-  } catch (/** @type {any} */ err) {
-    return err
-  }
 
   if ((err = validateDataSizeHash(msg))) return err
   if ((err = validateDomain(msg.metadata.domain))) return err
@@ -347,6 +358,7 @@ function validate(msg, tangle, sigkeys, msgID, rootID) {
     if ((err = validateTangle(msg, tangle, rootID))) return err
   }
   if ((err = validateSignature(msg))) return err
+  return undefined
 }
 
 module.exports = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pzp-db",
-  "version": "1.0.0",
+  "version": "1.0.4",
   "description": "Default PZP database",
   "homepage": "https://codeberg.org/pzp/pzp-db",
   "repository": {
@@ -24,6 +24,9 @@
     },
     "./msg-v4": {
       "require": "./lib/msg-v4/index.js"
+    },
+    "./db-tangle": {
+      "require": "./lib/db-tangle.js"
     }
   },
   "dependencies": {