1.0.4

Trying to fix https://codeberg.org/pzp/pzp-sync/issues/10

This doesn't solve the problem of missing rootMsg but maybe allows sync to fail more gracefully

Reviewed-on: https://codeberg.org/pzp/pzp-db/pulls/8
Co-authored-by: Jacob Karlsson <jacob.karlsson95@gmail.com>
Co-committed-by: Jacob Karlsson <jacob.karlsson95@gmail.com>

lib/db-tangle.js

@@ -0,0 +1,158 @@
+const pull = require('pull-stream')
+const p = require('node:util').promisify
+const MsgV4 = require('./msg-v4')
+
+/**
+ * @typedef {string} MsgID
+ * @typedef {import('./msg-v4').Msg} Msg
+ */
+
+/**
+ * @typedef {{
+ *   id?: never;
+ *   msg?: never;
+ *   received?: never;
+ * }} RecDeleted
+ *
+ * @typedef {{
+ *   id: MsgID;
+ *   msg: Msg;
+ *   received: number;
+ * }} RecPresent
+ *
+ * @typedef {RecPresent | RecDeleted} Rec
+ */
+
+/**
+ * @template T
+ * @typedef {[T] extends [void] ?
+ *   (...args: [Error] | []) => void :
+ *   (...args: [Error] | [null, T]) => void
+ * } CB
+ */
+
+class DBTangle extends MsgV4.Tangle {
+  /** @type {(msgID: MsgID, cb: CB<Msg>) => void} */
+  #getMsg
+
+  /**
+   * @param {MsgID} rootID
+   * @param {(msgID: MsgID, cb: CB<Msg>) => void} getMsg
+   */
+  constructor(rootID, getMsg) {
+    super(rootID)
+    this.#getMsg = getMsg
+  }
+
+  /**
+   * @param {MsgID} rootID
+   * @param {AsyncIterable<Rec>} recordsIter
+   * @param {(msgID: MsgID, cb: any) => void} getMsg
+   * @return {Promise<DBTangle>}
+   */
+  static async init(rootID, recordsIter, getMsg) {
+    const dbtangle = new DBTangle(rootID, getMsg)
+    for await (const rec of recordsIter) {
+      if (!rec.msg) continue
+      dbtangle.add(rec.id, rec.msg)
+    }
+    return dbtangle
+  }
+
+  /**
+   * Given a set of msgs (`msgIDs`) in this tangle, find all "deletable" and
+   * "erasable" msgs that precede that set.
+   *
+   * *Deletables* are msgs that precede `msgsIDs` but are not important in any
+   * validation path toward the root, and thus can be deleted.
+   *
+   * *Erasables* are msgs that precede `msgsIDs` and can be erased without
+   * losing a validation path toward the root.
+   * @param {Array<MsgID>} msgIDs
+   * @returns {{ deletables: Set<MsgID>, erasables: Set<MsgID> } | null}
+   */
+  getDeletablesAndErasables(...msgIDs) {
+    // Determine erasables
+    const erasables = new Set()
+    const minimum = this.getMinimumAmong(msgIDs)
+    for (const msgID of minimum) {
+      const trail = this.shortestPathToRoot(msgID)
+      if (!trail) return null
+      for (const id of trail) {
+        erasables.add(id)
+      }
+    }
+
+    // Determine deletables
+    const deletables = new Set()
+    const sorted = this.topoSort()
+    for (const msgID of sorted) {
+      if (erasables.has(msgID)) continue
+      if (minimum.some((min) => this.precedes(msgID, min))) {
+        deletables.add(msgID)
+      }
+    }
+
+    return { deletables, erasables }
+  }
+
+  /**
+   * @param {Array<string>=} minSet
+   * @param {Array<string>=} maxSet
+   * @param {CB<Array<Msg>>=} cb
+   * @return {Promise<Array<Msg>>|void}
+   */
+  slice(minSet = [], maxSet = [], cb) {
+    // @ts-ignore
+    if (cb === undefined) return p(this.slice).bind(this)(minSet, maxSet)
+
+    const minSetGood = minSet.filter((msgID) => this.has(msgID))
+    const maxSetGood = maxSet.filter((msgID) => this.has(msgID))
+    const minSetTight = this.getMinimumAmong(minSetGood)
+
+    const trail = new Set()
+    for (const msgID of minSetTight) {
+      const path = this.shortestPathToRoot(msgID)
+      if (!path) return cb(Error("Couldn't get shortest path to root when slicing dbtangle"))
+      for (const msgID of path) {
+        trail.add(msgID)
+      }
+    }
+
+    const msgs = /**@type {Array<Msg>}*/ ([])
+
+    pull(
+      pull.values(this.topoSort()),
+      pull.asyncMap((msgID, cb) => {
+        this.#getMsg(msgID, (err, msg) => {
+          if (err) return cb(err)
+          cb(null, { id: msgID, msg })
+        })
+      }),
+      pull.drain(
+        (rec) => {
+          if (trail.has(rec.id)) {
+            if (rec.msg) msgs.push({ ...rec.msg, data: null })
+          }
+          const isMin = minSetGood.includes(rec.id)
+          const isMax = maxSetGood.includes(rec.id)
+          const isBeforeMin = minSetGood.some((min) =>
+            this.precedes(rec.id, min)
+          )
+          const isAfterMax = maxSetGood.some((max) =>
+            this.precedes(max, rec.id)
+          )
+          if (!isMin && isBeforeMin) return
+          if (!isMax && isAfterMax) return
+          if (rec.msg) msgs.push(rec.msg)
+        },
+        (err) => {
+          if (err) return cb(Error('DBTangle.slice() failed', { cause: err }))
+          return cb(null, msgs)
+        }
+      )
+    )
+  }
+}
+
+module.exports = DBTangle

lib/index.js

@@ -8,6 +8,7 @@ const pull = require('pull-stream')
 const p = require('node:util').promisify
 const Log = require('./log')
 const MsgV4 = require('./msg-v4')
+const DBTangle = require('./db-tangle')
 const {
   SIGNATURE_TAG_ACCOUNT_ADD,
   ACCOUNT_SELF,
@@ -93,128 +94,6 @@ function assertValidConfig(config) {
   }
 }
 
-class DBTangle extends MsgV4.Tangle {
-  /** @type {(msgID: MsgID, cb: CB<Msg>) => void} */
-  #getMsg
-
-  /**
-   * @param {MsgID} rootID
-   * @param {(msgID: MsgID, cb: CB<Msg>) => void} getMsg
-   */
-  constructor(rootID, getMsg) {
-    super(rootID)
-    this.#getMsg = getMsg
-  }
-
-  /**
-   * @param {MsgID} rootID
-   * @param {AsyncIterable<Rec>} recordsIter
-   * @param {(msgID: MsgID, cb: any) => void} getMsg
-   * @return {Promise<DBTangle>}
-   */
-  static async init(rootID, recordsIter, getMsg) {
-    const dbtangle = new DBTangle(rootID, getMsg)
-    for await (const rec of recordsIter) {
-      if (!rec.msg) continue
-      dbtangle.add(rec.id, rec.msg)
-    }
-    return dbtangle
-  }
-
-  /**
-   * Given a set of msgs (`msgIDs`) in this tangle, find all "deletable" and
-   * "erasable" msgs that precede that set.
-   *
-   * *Deletables* are msgs that precede `msgsIDs` but are not important in any
-   * validation path toward the root, and thus can be deleted.
-   *
-   * *Erasables* are msgs that precede `msgsIDs` and can be erased without
-   * losing a validation path toward the root.
-   * @param {Array<MsgID>} msgIDs
-   * @returns {{ deletables: Set<MsgID>, erasables: Set<MsgID> }}
-   */
-  getDeletablesAndErasables(...msgIDs) {
-    // Determine erasables
-    const erasables = new Set()
-    const minimum = this.getMinimumAmong(msgIDs)
-    for (const msgID of minimum) {
-      const trail = this.shortestPathToRoot(msgID)
-      for (const id of trail) {
-        erasables.add(id)
-      }
-    }
-
-    // Determine deletables
-    const deletables = new Set()
-    const sorted = this.topoSort()
-    for (const msgID of sorted) {
-      if (erasables.has(msgID)) continue
-      if (minimum.some((min) => this.precedes(msgID, min))) {
-        deletables.add(msgID)
-      }
-    }
-
-    return { deletables, erasables }
-  }
-
-  /**
-   * @param {Array<string>=} minSet
-   * @param {Array<string>=} maxSet
-   * @param {CB<Array<Msg>>=} cb
-   * @return {Promise<Array<Msg>>|void}
-   */
-  slice(minSet = [], maxSet = [], cb) {
-    // @ts-ignore
-    if (cb === undefined) return p(this.slice).bind(this)(minSet, maxSet)
-
-    const minSetGood = minSet.filter((msgID) => this.has(msgID))
-    const maxSetGood = maxSet.filter((msgID) => this.has(msgID))
-    const minSetTight = this.getMinimumAmong(minSetGood)
-
-    const trail = new Set()
-    for (const msgID of minSetTight) {
-      const path = this.shortestPathToRoot(msgID)
-      for (const msgID of path) {
-        trail.add(msgID)
-      }
-    }
-
-    const msgs = /**@type {Array<Msg>}*/ ([])
-
-    pull(
-      pull.values(this.topoSort()),
-      pull.asyncMap((msgID, cb) => {
-        this.#getMsg(msgID, (err, msg) => {
-          if (err) return cb(err)
-          cb(null, { id: msgID, msg })
-        })
-      }),
-      pull.drain(
-        (rec) => {
-          if (trail.has(rec.id)) {
-            if (rec.msg) msgs.push({ ...rec.msg, data: null })
-          }
-          const isMin = minSetGood.includes(rec.id)
-          const isMax = maxSetGood.includes(rec.id)
-          const isBeforeMin = minSetGood.some((min) =>
-            this.precedes(rec.id, min)
-          )
-          const isAfterMax = maxSetGood.some((max) =>
-            this.precedes(max, rec.id)
-          )
-          if (!isMin && isBeforeMin) return
-          if (!isMax && isAfterMax) return
-          if (rec.msg) msgs.push(rec.msg)
-        },
-        (err) => {
-          if (err) return cb(Error('DBTangle.slice() failed', { cause: err }))
-          return cb(null, msgs)
-        }
-      )
-    )
-  }
-}
-
 /**
  * @param {Peer} peer
  * @param {Config} config
@@ -529,16 +408,17 @@ function initDB(peer, config) {
           rec.msg.metadata.accountTips,
           (err, sigkeys) => {
             if (err) return cb(err)
-
+            // TODO: how is sigkeys able to be undefined here? typechecker why are you weird?
+            if (!sigkeys) return cb(Error("Sigkeys missing somehow"))
             // Don't accept ghosts to come back, unless they are trail msgs
             if (!!rec.msg.data && ghosts.read(tangleID).has(rec.id)) {
               return cb(Error('Refusing a ghost msg to come back'))
             }
 
-            if (
+            
-              (err = MsgV4.validate(rec.msg, tangle, sigkeys, rec.id, tangleID))
+            const valErr = MsgV4.validate(rec.msg, tangle, sigkeys, rec.id, tangleID)
-            ) {
+            if (valErr) {
-              return cb(Error('Invalid msg', { cause: err }))
+              return cb(Error('Invalid msg', { cause: valErr }))
             }
 
             /** @param {CB<void>} cb */

lib/msg-v4/tangle.js

@@ -222,11 +222,12 @@ class Tangle {
   }
 
   /**
-   * @returns {'feed' | 'account' | 'weave'}
+   * @returns {'feed' | 'account' | 'weave' | null}
    */
   get type() {
     if (!this.#rootMsg) {
-      throw new Error(`Tangle "${this.#rootID}" is missing root message`)
+      console.trace(new Error(`Tangle "${this.#rootID}" is missing root message`))
+      return null
     }
     if (this.#isFeed()) return 'feed'
     if (this.#rootMsg.metadata.account === 'self') return 'account'
@@ -235,7 +236,8 @@ class Tangle {
 
   get root() {
     if (!this.#rootMsg) {
-      throw new Error(`Tangle "${this.#rootID}" is missing root message`)
+      console.trace(new Error(`Tangle "${this.#rootID}" is missing root message`))
+      return null
     }
     return this.#rootMsg
   }
@@ -256,7 +258,8 @@ class Tangle {
       if (!prev) break
       if (prev === lastPrev) {
         // prettier-ignore
-        throw new Error(`Tangle "${this.#rootID}" has a cycle or lacking a trail to root`)
+        console.trace(new Error(`Tangle "${this.#rootID}" has a cycle or lacking a trail to root`))
+        return null
       } else {
         lastPrev = prev
       }

lib/msg-v4/validation.js

@@ -107,6 +107,8 @@ function validateSigkeyAndAccount(msg, tangle, sigkeys) {
       // prettier-ignore
       return `invalid msg: accountTips "${msg.metadata.accountTips}" should have been null in an account tangle\n` + JSON.stringify(msg)
     }
+  } else if (tangle.type === null) {
+    return "Cannot validate tangle of unknown type"
   }
   return undefined
 }
@@ -199,6 +201,8 @@ function validateTangle(msg, tangle, tangleID) {
       // prettier-ignore
       return `invalid msg: account "${msg.metadata.account}" should have been feed account "${account}"\n` + JSON.stringify(msg)
     }
+  } else if (tangle.type === null) {
+    return "Unknown tangle type"
   }
   let lastPrev = null
   let minDiff = Infinity
@@ -342,11 +346,8 @@ function validate(msg, tangle, sigkeys, msgID, rootID) {
   if ((err = validateSigkey(msg))) return err
   if ((err = validateData(msg))) return err
 
-  try {
+  if (tangle.type === 'feed' && isMoot(msg)) return // nothing else to check
-    if (tangle.type === 'feed' && isMoot(msg)) return // nothing else to check
+  if (tangle.type === null) return "Missing tangle type when validating msg"
-  } catch (/** @type {any} */ err) {
-    return err
-  }
 
   if ((err = validateDataSizeHash(msg))) return err
   if ((err = validateDomain(msg.metadata.domain))) return err
@@ -357,6 +358,7 @@ function validate(msg, tangle, sigkeys, msgID, rootID) {
     if ((err = validateTangle(msg, tangle, rootID))) return err
   }
   if ((err = validateSignature(msg))) return err
+  return undefined
 }
 
 module.exports = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pzp-db",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Default PZP database",
   "homepage": "https://codeberg.org/pzp/pzp-db",
   "repository": {
@@ -24,6 +24,9 @@
     },
     "./msg-v4": {
       "require": "./lib/msg-v4/index.js"
+    },
+    "./db-tangle": {
+      "require": "./lib/db-tangle.js"
     }
   },
   "dependencies": {