validation on add() supports encrypted inner msgs

lib/encryption.js

@@ -40,13 +40,15 @@ function keypairToSSBKeys(keypair) {
   }
 }
 
+const decryptCache = new WeakMap()
+
 /**
- * @param {RecPresent} rec
+ * @param {Pick<RecPresent, 'msg'> & Partial<Pick<RecPresent, 'id' | 'misc' | 'received'>>} rec
  * @param {any} peer
  * @param {any} config
- * @returns {RecPresent}
  */
 function decrypt(rec, peer, config) {
+  if (decryptCache.has(rec)) return decryptCache.get(rec)
   const msgEncrypted = rec.msg
   const { data } = msgEncrypted
   if (typeof data !== 'string') return rec
@@ -63,7 +65,7 @@ function decrypt(rec, peer, config) {
   // Reconstruct KVT in JS encoding
   const msgDecrypted = MsgV3.fromPlaintextBuffer(plaintextBuf, msgEncrypted)
 
-  return {
+  const recDecrypted = {
     id: rec.id,
     msg: msgDecrypted,
     received: rec.received,
@@ -74,6 +76,8 @@ function decrypt(rec, peer, config) {
       encryptionFormat: encryptionFormat.name,
     },
   }
+  decryptCache.set(rec, recDecrypted)
+  return recDecrypted
 }
 
 /**

lib/index.js

@@ -13,6 +13,7 @@ const MsgV3 = require('./msg-v3')
 const {
   SIGNATURE_TAG_ACCOUNT_ADD,
   ACCOUNT_SELF,
+  ACCOUNT_ANY,
 } = require('./msg-v3/constants')
 const { ReadyGate } = require('./utils')
 const { decrypt } = require('./encryption')
@@ -197,19 +198,15 @@ function initDB(peer, config) {
         seq: 0,
       },
     }
-    log.append(
+    log.append(rec, (/** @type {any} */ err, /** @type {number} */ offset) => {
-      rec,
+      if (err) return cb(new Error('logAppend failed', { cause: err }))
-      (/** @type {any} */ err, /** @type {number} */ newOffset) => {
+      const size = b4a.from(JSON.stringify(rec), 'utf8').length
-        if (err) return cb(new Error('logAppend failed', { cause: err }))
+      const seq = recs.length
-        const offset = newOffset // latestOffset
+      const recExposed = decrypt(rec, peer, config)
-        const size = b4a.from(JSON.stringify(rec), 'utf8').length
+      rec.misc = recExposed.misc = { offset, size, seq }
-        const seq = recs.length
+      recs.push(recExposed)
-        const recExposed = decrypt(rec, peer, config)
+      cb(null, rec)
-        rec.misc = recExposed.misc = { offset, size, seq }
+    })
-        recs.push(recExposed)
-        cb(null, rec)
-      }
-    )
   }
 
   /**
@@ -256,6 +253,25 @@ function initDB(peer, config) {
     return tangles
   }
 
+  /**
+   * @param {Pick<RecPresent, 'id' | 'msg'>} rec
+   * @returns {Tangle | null}
+   */
+  function getAccountTangle(rec) {
+    const accountID = getAccountID(rec)
+    let accountTangle = /** @type {Tangle | null} */ (null)
+    if (accountID) {
+      accountTangle = new DBTangle(accountID, records())
+      if (rec.id === accountID) {
+        accountTangle.add(rec.id, rec.msg)
+      }
+      if (!accountTangle.has(accountID)) {
+        throw new Error(`Account tangle "${accountID}" is locally unknown`)
+      }
+    }
+    return accountTangle
+  }
+
   /**
    * Find which pubkeys are authorized to sign this msg given the account.
    *
@@ -288,6 +304,65 @@ function initDB(peer, config) {
     scannedLog.onReady(cb)
   }
 
+  /**
+   * Checks whether the given `rec` can correctly fit into the log, validating
+   * the msg in relation to the given `tangleID`, and whether the account is
+   * locally known.
+   *
+   * @param {Pick<RecPresent, 'id' | 'msg'>} rec
+   * @param {string} tangleID
+   * @returns {Error | null}
+   */
+  function verifyRec(rec, tangleID) {
+    // TODO: optimize this. This may be slow if you're adding many msgs in a
+    // row, because it creates a new Map() each time. Perhaps with QuickLRU
+    const tangle = new DBTangle(tangleID, records())
+    if (rec.id === tangleID) {
+      tangle.add(rec.id, rec.msg)
+    }
+
+    // Identify the account and its pubkeys:
+    /** @type {Tangle | null} */
+    let accountTangle
+    try {
+      accountTangle = getAccountTangle(rec)
+    } catch (err) {
+      // prettier-ignore
+      return new Error('Failed to identify the account of this msg', { cause: err })
+    }
+    const pubkeys = getPubkeysInAccount(accountTangle)
+
+    let err
+    if ((err = MsgV3.validate(rec.msg, tangle, pubkeys, rec.id, tangleID))) {
+      return new Error('Failed msg validation', { cause: err })
+    }
+
+    // Account tangle related validations
+    if (rec.msg.metadata.account === ACCOUNT_SELF) {
+      const validAccountTangle = /** @type {Tangle} */ (accountTangle)
+      if ((err = validateAccountMsg(rec.msg, validAccountTangle))) {
+        return new Error('Failed msg account validation', { cause: err })
+      }
+    }
+
+    // Unwrap encrypted inner msg and verify it too
+    if (typeof rec.msg.data === 'string') {
+      const recDecrypted = decrypt(rec, peer, config)
+      if (MsgV3.isMsg(recDecrypted.msg.data)) {
+        const innerMsg = /** @type {Msg} */ (recDecrypted.msg.data)
+        const innerMsgID = MsgV3.getMsgID(innerMsg)
+        const innerRec = { id: innerMsgID, msg: innerMsg }
+        try {
+          verifyRec(innerRec, innerMsgID)
+        } catch (err) {
+          return new Error('Failed to verify inner msg', { cause: err })
+        }
+      }
+    }
+
+    return null
+  }
+
   /**
    * @param {Msg} msg
    * @param {string} tangleID
@@ -302,39 +377,9 @@ function initDB(peer, config) {
     if ((rec = getRecord(msgID))) return cb(null, rec)
     else rec = { msg, id: msgID }
 
-    // TODO: optimize this. This may be slow if you're adding many msgs in a
-    // row, because it creates a new Map() each time. Perhaps with QuickLRU
-    const tangle = new DBTangle(tangleID, records())
-    if (msgID === tangleID) {
-      tangle.add(msgID, msg)
-    }
-
-    // Identify the account and its pubkeys:
-    const accountID = getAccountID(rec)
-    let accountTangle = /** @type {Tangle | null} */ (null)
-    if (accountID) {
-      accountTangle = new DBTangle(accountID, records())
-      if (msgID === accountID) {
-        accountTangle.add(msgID, msg)
-      }
-      if (!accountTangle.has(accountID)) {
-        // prettier-ignore
-        return cb(new Error('add() failed because the account tangle is unknown'))
-      }
-    }
-    const pubkeys = getPubkeysInAccount(accountTangle)
-
     let err
-    if ((err = MsgV3.validate(msg, tangle, pubkeys, msgID, tangleID))) {
+    if ((err = verifyRec(rec, tangleID))) {
-      return cb(new Error('add() failed msg validation', { cause: err }))
+      return cb(new Error('add() failed to verify msg', { cause: err }))
-    }
-
-    // Account tangle related validations
-    if (msg.metadata.account === ACCOUNT_SELF) {
-      if ((err = validateAccountMsg(msg, accountTangle))) {
-        // prettier-ignore
-        return cb(new Error('add() failed msg account validation', { cause: err }))
-      }
     }
 
     logAppend(msgID, msg, (err, rec) => {
@@ -346,13 +391,10 @@ function initDB(peer, config) {
 
   /**
    * @param {Msg} msg
-   * @param {Tangle | null} accountTangle
+   * @param {Tangle} accountTangle
    * @returns {string | undefined}
    */
   function validateAccountMsg(msg, accountTangle) {
-    if (!accountTangle) {
-      return 'invalid account msg: account tangle is unknown'
-    }
     if (!MsgV3.isRoot(msg)) {
       /** @type {AccountData} */
       const data = msg.data
@@ -395,16 +437,15 @@ function initDB(peer, config) {
    * @returns {string | null}
    */
   function getAccountID(rec) {
-    if (!rec.msg) return null
     if (rec.msg.metadata.account === ACCOUNT_SELF) {
       for (const tangleID in rec.msg.metadata.tangles) {
         return tangleID
       }
       return rec.id
-    } else if (rec.msg.metadata.account) {
+    } else if (rec.msg.metadata.account === ACCOUNT_ANY) {
-      return rec.msg.metadata.account
-    } else {
       return null
+    } else {
+      return rec.msg.metadata.account
     }
   }
 

lib/msg-v3/index.js

@@ -14,6 +14,7 @@ const {
   validateDomain,
   validateData,
   validate,
+  validateShape,
   validateMsgID,
 } = require('./validation')
 const Tangle = require('./tangle')
@@ -277,7 +278,16 @@ function isRoot(msg) {
   return isEmptyObject(msg.metadata.tangles)
 }
 
+/**
+ * @param {any} x
+ * @returns {x is Msg}
+ */
+function isMsg(x) {
+  return !validateShape(x)
+}
+
 module.exports = {
+  isMsg,
   isMoot,
   isRoot,
   getMsgID,

lib/msg-v3/validation.js

@@ -342,6 +342,7 @@ function validate(msg, tangle, pubkeys, msgID, rootID) {
 module.exports = {
   validateDomain,
   validateData,
+  validateShape,
   validate,
   validateMsgID,
 }

protospec.md

@@ -41,6 +41,8 @@ interface Msg {
 }
 ```
 
+**Depth:** we NEED this field because it is the most reliable way of calculating lipmaa distances between msgs, in the face of sliced replication. For example, given that older messages (except the certificate pool) would be deleted, the "graph depth" calculation for a msg may change over time, but we need a way of keeping this calculation stable and deterministic.
+
 ## Account tangle msgs
 
 Msgs in an account tangle are special because they have empty `account` and `accountTips` fields.

test/account-add.test.js

@@ -144,7 +144,7 @@ test('keypair with no "add" powers cannot account.add()', async (t) => {
   // Test replicator-side power validation
   assert.rejects(
     p(peer1again.db.add)(msg3, id),
-    /add\(\) failed msg account validation/
+    /add\(\) failed to verify msg/
   )
 
   await p(peer1again.close)()