tweak tests, include dagfeed

lib/feed-v1/get-msg-id.js

@@ -0,0 +1,26 @@
+const blake3 = require('blake3')
+const base58 = require('bs58')
+const stringify = require('fast-json-stable-stringify')
+
+function getMsgHashBuf(nativeMsg) {
+  const { metadata, signature } = nativeMsg
+  const metadataBuf = Buffer.from(stringify(metadata), 'utf8')
+  const sigBuf = base58.decode(signature)
+  return blake3
+    .hash(Buffer.concat([metadataBuf, sigBuf]))
+    .subarray(0, 16)
+}
+
+function getMsgHash(nativeMsg) {
+  const msgHashBuf = getMsgHashBuf(nativeMsg)
+  return base58.encode(msgHashBuf)
+}
+
+function getMsgId(nativeMsg) {
+  const author = nativeMsg.metadata.author
+  const type = nativeMsg.metadata.type
+  const msgHash = getMsgHash(nativeMsg)
+  return `ssb:message/dag/${author}/${type}/${msgHash}`
+}
+
+module.exports = { getMsgId, getMsgHash }

lib/feed-v1/index.js

@@ -0,0 +1,167 @@
+// SPDX-FileCopyrightText: 2022 Andre 'Staltz' Medeiros
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+const stringify = require('fast-json-stable-stringify')
+const ed25519 = require('ssb-keys/sodium')
+const base58 = require('bs58')
+const {
+  stripAuthor,
+  stripMsgKey,
+  unstripMsgKey,
+  unstripAuthor,
+} = require('./strip')
+const { getMsgId, getMsgHash } = require('./get-msg-id')
+const representContent = require('./represent-content')
+const {
+  validateType,
+  validateContent,
+  validate,
+  validateOOO,
+  validateBatch,
+  validateOOOBatch,
+} = require('./validation')
+
+const name = 'dag'
+const encodings = ['js']
+
+function getFeedId(nativeMsg) {
+  return nativeMsg.metadata.author + nativeMsg.metadata.type
+}
+
+function getSequence(nativeMsg) {
+  throw new Error('getSequence not supported for dagfeed')
+}
+
+function isNativeMsg(x) {
+  return (
+    typeof x === 'object' &&
+    !!x &&
+    typeof x.metadata.author === 'string' &&
+    x.metadata.author &&
+    typeof x.metadata.type === 'string' &&
+    x.metadata.type
+  )
+}
+
+function isAuthor(author) {
+  if (typeof author !== 'string') return false
+  return author.startsWith('ssb:feed/dag/')
+}
+
+function toPlaintextBuffer(opts) {
+  return Buffer.from(stringify(opts.content), 'utf8')
+}
+
+function newNativeMsg(opts) {
+  let err
+  if ((err = validateType(opts.type))) throw err
+  if (opts.previous && !Array.isArray(opts.previous)) {
+    // prettier-ignore
+    throw new Error('opts.previous must be an array, but got ' + typeof opts.previous)
+  }
+
+  const [contentHash, contentSize] = representContent(opts.content)
+  const nativeMsg = {
+    metadata: {
+      author: stripAuthor(opts.keys.id),
+      type: opts.type,
+      previous: (opts.previous ?? []).map(stripMsgKey),
+      timestamp: +opts.timestamp,
+      contentHash,
+      contentSize,
+    },
+    content: opts.content,
+    signature: '',
+  }
+  if ((err = validateContent(nativeMsg))) throw err
+
+  const metadataBuf = Buffer.from(stringify(nativeMsg.metadata), 'utf8')
+  // FIXME: this should allow using hmacKey
+  const privateKey = Buffer.from(opts.keys.private, 'base64')
+  const signature = ed25519.sign(privateKey, metadataBuf)
+  nativeMsg.signature = base58.encode(signature)
+  return nativeMsg
+}
+
+function fromNativeMsg(nativeMsg, encoding = 'js') {
+  if (encoding === 'js') {
+    const msgVal = {
+      // traditional:
+      previous: nativeMsg.metadata.previous.map((id) =>
+        unstripMsgKey(nativeMsg, id)
+      ),
+      sequence: 0,
+      author: unstripAuthor(nativeMsg),
+      timestamp: nativeMsg.metadata.timestamp,
+      content: nativeMsg.content,
+      signature: nativeMsg.signature,
+      // unusual:
+      contentHash: nativeMsg.metadata.contentHash,
+      contentSize: nativeMsg.metadata.contentSize,
+      type: nativeMsg.metadata.type,
+    }
+    if (typeof msgVal.content === 'object') {
+      msgVal.content.type = nativeMsg.metadata.type
+    }
+    return msgVal
+  } else {
+    // prettier-ignore
+    throw new Error(`Feed format "${name}" does not support encoding "${encoding}"`)
+  }
+}
+
+function fromDecryptedNativeMsg(plaintextBuf, nativeMsg, encoding = 'js') {
+  if (encoding === 'js') {
+    const msgVal = fromNativeMsg(nativeMsg, 'js')
+    const content = JSON.parse(plaintextBuf.toString('utf8'))
+    msgVal.content = content
+    msgVal.content.type = nativeMsg.metadata.type
+    return msgVal
+  } else {
+    // prettier-ignore
+    throw new Error(`Feed format "${name}" does not support encoding "${encoding}"`)
+  }
+}
+
+function toNativeMsg(msgVal, encoding = 'js') {
+  if (encoding === 'js') {
+    return {
+      metadata: {
+        author: stripAuthor(msgVal.author),
+        type: msgVal.type ?? '',
+        previous: (msgVal.previous ?? []).map(stripMsgKey),
+        timestamp: msgVal.timestamp,
+        contentHash: msgVal.contentHash,
+        contentSize: msgVal.contentSize,
+      },
+      content: msgVal.content,
+      signature: msgVal.signature,
+    }
+  } else {
+    // prettier-ignore
+    throw new Error(`Feed format "${name}" does not support encoding "${encoding}"`)
+  }
+}
+
+module.exports = {
+  name,
+  encodings,
+  getMsgId,
+  getFeedId,
+  getSequence,
+  isAuthor,
+  isNativeMsg,
+  toPlaintextBuffer,
+  newNativeMsg,
+  fromNativeMsg,
+  fromDecryptedNativeMsg,
+  toNativeMsg,
+  validate,
+  validateOOO,
+  validateBatch,
+  validateOOOBatch,
+
+  // custom APIs:
+  getMsgHash,
+}

lib/feed-v1/represent-content.js

@@ -0,0 +1,12 @@
+const blake3 = require('blake3')
+const base58 = require('bs58')
+const stringify = require('fast-json-stable-stringify')
+
+function representContent(content) {
+  const contentBuf = Buffer.from(stringify(content), 'utf8')
+  const hash = base58.encode(blake3.hash(contentBuf).subarray(0, 16))
+  const size = contentBuf.length
+  return [hash, size]
+}
+
+module.exports = representContent

lib/feed-v1/strip.js

@@ -0,0 +1,31 @@
+function stripMsgKey(msgKey) {
+  if (typeof msgKey === 'object') return stripMsgKey(msgKey.key)
+  if (msgKey.startsWith('ssb:message/dag/')) {
+    const parts = msgKey.split('/')
+    return parts[parts.length - 1]
+  } else {
+    return msgKey
+  }
+}
+
+function unstripMsgKey(nativeMsg, msgId) {
+  const { author, type } = nativeMsg.metadata
+  return `ssb:message/dag/${author}/${type}/${msgId}`
+}
+
+function stripAuthor(id) {
+  const withoutPrefix = id.replace('ssb:feed/dag/', '')
+  return withoutPrefix.split('/')[0]
+}
+
+function unstripAuthor(nativeMsg) {
+  const { author, type } = nativeMsg.metadata
+  return `ssb:feed/dag/${author}/${type}`
+}
+
+module.exports = {
+  stripMsgKey,
+  unstripMsgKey,
+  stripAuthor,
+  unstripAuthor,
+}

lib/feed-v1/validation.js

@@ -0,0 +1,266 @@
+const base58 = require('bs58')
+const ed25519 = require('ssb-keys/sodium')
+const stringify = require('fast-json-stable-stringify')
+const { stripMsgKey } = require('./strip')
+const { getMsgHash } = require('./get-msg-id')
+
+function validateShape(nativeMsg) {
+  if (!nativeMsg || typeof nativeMsg !== 'object') {
+    return new Error('invalid message: not a dag msg')
+  }
+  if (!nativeMsg.metadata || typeof nativeMsg.metadata !== 'object') {
+    return new Error('invalid message: must have metadata')
+  }
+  if (typeof nativeMsg.metadata.author === 'undefined') {
+    return new Error('invalid message: must have metadata.author')
+  }
+  if (typeof nativeMsg.metadata.type === 'undefined') {
+    return new Error('invalid message: must have metadata.sequence')
+  }
+  if (typeof nativeMsg.metadata.previous === 'undefined') {
+    return new Error('invalid message: must have metadata.previous')
+  }
+  if (typeof nativeMsg.metadata.timestamp === 'undefined') {
+    return new Error('invalid message: must have metadata.timestamp')
+  }
+  if (typeof nativeMsg.metadata.contentHash === 'undefined') {
+    return new Error('invalid message: must have metadata.contentHash')
+  }
+  if (typeof nativeMsg.metadata.contentSize === 'undefined') {
+    return new Error('invalid message: must have metadata.contentSize')
+  }
+  if (typeof nativeMsg.content === 'undefined') {
+    return new Error('invalid message: must have content')
+  }
+  if (typeof nativeMsg.signature === 'undefined') {
+    return new Error('invalid message: must have signature')
+  }
+}
+
+function validateAuthor(nativeMsg) {
+  try {
+    base58.decode(nativeMsg.metadata.author)
+  } catch (err) {
+    return new Error('invalid message: must have author as base58 string')
+  }
+}
+
+function validateSignature(nativeMsg, hmacKey) {
+  const { signature } = nativeMsg
+  if (typeof signature !== 'string') {
+    return new Error('invalid message: must have signature as a string')
+  }
+  try {
+    base58.decode(signature)
+  } catch (err) {
+    return new Error('invalid message: signature must be a base58 string')
+  }
+  const signatureBuf = Buffer.from(base58.decode(signature))
+  if (signatureBuf.length !== 64) {
+    // prettier-ignore
+    return new Error('invalid message: signature should be 64 bytes but was ' + signatureBuf.length + ', on feed: ' + nativeMsg.metadata.author);
+  }
+
+  const publicKeyBuf = Buffer.from(base58.decode(nativeMsg.metadata.author))
+  const signableBuf = Buffer.from(stringify(nativeMsg.metadata), 'utf8')
+  const verified = ed25519.verify(publicKeyBuf, signatureBuf, signableBuf)
+  if (!verified) {
+    // prettier-ignore
+    return new Error('invalid message: signature does not match, on feed: ' + nativeMsg.metadata.author);
+  }
+}
+
+function validatePrevious(nativeMsg, existingNativeMsgs) {
+  if (!Array.isArray(nativeMsg.metadata.previous)) {
+    // prettier-ignore
+    return new Error('invalid message: previous must be an array, on feed: ' + nativeMsg.metadata.author);
+  }
+  for (const prevId of nativeMsg.metadata.previous) {
+    if (typeof prevId !== 'string') {
+      // prettier-ignore
+      return new Error('invalid message: previous must contain strings but found ' + prevId + ', on feed: ' + nativeMsg.metadata.author);
+    }
+    if (prevId.startsWith('ssb:')) {
+      // prettier-ignore
+      return new Error('invalid message: previous must not contain SSB URIs, on feed: ' + nativeMsg.metadata.author);
+    }
+
+    if (existingNativeMsgs instanceof Set) {
+      if (!existingNativeMsgs.has(prevId)) {
+        // prettier-ignore
+        return new Error('invalid message: previous ' + prevId + ' is not a known message ID, on feed: ' + nativeMsg.metadata.author);
+      }
+      continue
+    } else {
+      let found = false
+      for (const nmsg of existingNativeMsgs) {
+        const existingId = nmsg.key
+          ? stripMsgKey(nmsg.key)
+          : typeof nmsg === 'string'
+          ? stripMsgKey(nmsg)
+          : getMsgHash(nmsg)
+        if (existingId === prevId) {
+          found = true
+          break
+        }
+      }
+      if (!found) {
+        // prettier-ignore
+        return new Error('invalid message: previous ' + prevId + ' is not a known message ID, on feed: ' + nativeMsg.metadata.author);
+      }
+    }
+  }
+}
+
+function validateFirstPrevious(nativeMsg) {
+  if (!Array.isArray(nativeMsg.metadata.previous)) {
+    // prettier-ignore
+    return new Error('invalid message: previous must be an array, on feed: ' + nativeMsg.metadata.author);
+  }
+  if (nativeMsg.metadata.previous.length !== 0) {
+    // prettier-ignore
+    return new Error('initial message: previous must be an empty array, on feed: ' + nativeMsg.metadata.author);
+  }
+}
+
+function validateTimestamp(nativeMsg) {
+  if (typeof nativeMsg.metadata.timestamp !== 'number') {
+    // prettier-ignore
+    return new Error('initial message must have timestamp, on feed: ' + nativeMsg.metadata.author);
+  }
+}
+
+function validateType(type) {
+  if (!type || typeof type !== 'string') {
+    // prettier-ignore
+    return new Error('type is not a string');
+  }
+  if (type.length > 100) {
+    // prettier-ignore
+    return new Error('invalid type ' + type + ' is 100+ characters long');
+  }
+  if (type.length < 3) {
+    // prettier-ignore
+    return new Error('invalid type ' + type + ' is shorter than 3 characters');
+  }
+  if (/[^a-zA-Z0-9_]/.test(type)) {
+    // prettier-ignore
+    return new Error('invalid type ' + type + ' contains characters other than a-z, A-Z, 0-9, or _');
+  }
+}
+
+function validateContent(nativeMsg) {
+  const { content } = nativeMsg
+  if (!content) {
+    return new Error('invalid message: must have content')
+  }
+  if (Array.isArray(content)) {
+    return new Error('invalid message: content must not be an array')
+  }
+  if (typeof content !== 'object' && typeof content !== 'string') {
+    // prettier-ignore
+    return new Error('invalid message: content must be an object or string, on feed: ' + nativeMsg.metadata.author);
+  }
+}
+
+function validateHmac(hmacKey) {
+  if (!hmacKey) return
+  if (typeof hmacKey !== 'string' && !Buffer.isBuffer(hmacKey)) {
+    return new Error('invalid hmac key: must be a string or buffer')
+  }
+  const bytes = Buffer.isBuffer(hmacKey)
+    ? hmacKey
+    : Buffer.from(hmacKey, 'base64')
+
+  if (typeof hmacKey === 'string' && bytes.toString('base64') !== hmacKey) {
+    return new Error('invalid hmac')
+  }
+
+  if (bytes.length !== 32) {
+    return new Error('invalid hmac, it should have 32 bytes')
+  }
+}
+
+function emptyExisting(existingNativeMsgs) {
+  if (existingNativeMsgs instanceof Set) {
+    return existingNativeMsgs.size === 0
+  } else if (Array.isArray(existingNativeMsgs)) {
+    return existingNativeMsgs.length === 0
+  } else {
+    return !existingNativeMsgs
+  }
+}
+
+function validateSync(nativeMsg, existingNativeMsgs, hmacKey) {
+  let err
+  if ((err = validateShape(nativeMsg))) return err
+  if ((err = validateHmac(hmacKey))) return err
+  if ((err = validateAuthor(nativeMsg))) return err
+  if ((err = validateTimestamp(nativeMsg))) return err
+  if (emptyExisting(existingNativeMsgs)) {
+    if ((err = validateFirstPrevious(nativeMsg))) return err
+  } else {
+    if ((err = validatePrevious(nativeMsg, existingNativeMsgs))) return err
+  }
+  if ((err = validateContent(nativeMsg))) return err
+  if ((err = validateSignature(nativeMsg, hmacKey))) return err
+}
+
+// function validateOOOSync(nativeMsg, hmacKey) {
+//   let err
+//   if ((err = validateShape(nativeMsg))) return err
+//   if ((err = validateHmac(hmacKey))) return err
+//   if ((err = validateAuthor(nativeMsg))) return err
+//   if ((err = validateHash(nativeMsg))) return err
+//   if ((err = validateTimestamp(nativeMsg))) return err
+//   if ((err = validateOrder(nativeMsg))) return err
+//   if ((err = validateContent(nativeMsg))) return err
+//   if ((err = validateAsJSON(nativeMsg))) return err
+//   if ((err = validateSignature(nativeMsg, hmacKey))) return err
+// }
+
+function validate(nativeMsg, prevNativeMsg, hmacKey, cb) {
+  let err
+  if ((err = validateSync(nativeMsg, prevNativeMsg, hmacKey))) {
+    return cb(err)
+  }
+  cb()
+}
+
+// function validateOOO(nativeMsg, hmacKey, cb) {
+//   let err
+//   if ((err = validateOOOSync(nativeMsg, hmacKey))) {
+//     return cb(err)
+//   }
+//   cb()
+// }
+
+// function validateBatch(nativeMsgs, prevNativeMsg, hmacKey, cb) {
+//   let err
+//   let prev = prevNativeMsg
+//   for (const nativeMsg of nativeMsgs) {
+//     err = validateSync(nativeMsg, prev, hmacKey)
+//     if (err) return cb(err)
+//     prev = nativeMsg
+//   }
+//   cb()
+// }
+
+// function validateOOOBatch(nativeMsgs, hmacKey, cb) {
+//   let err
+//   for (const nativeMsg of nativeMsgs) {
+//     err = validateOOOSync(nativeMsg, hmacKey)
+//     if (err) return cb(err)
+//   }
+//   cb()
+// }
+
+module.exports = {
+  validateType,
+  validateContent,
+
+  validate,
+  // validateBatch,
+  // validateOOO,
+  // validateOOOBatch,
+}

package.json

@@ -20,6 +20,9 @@
   },
   "dependencies": {
     "async-append-only-log": "^4.3.10",
+    "blake3": "^2.1.7",
+    "bs58": "^5.0.0",
+    "fast-json-stable-stringify": "^2.1.0",
     "obz": "^1.1.0",
     "promisify-4loc": "^1.0.0",
     "push-stream": "^11.2.0"

test/add.test.js

@@ -8,7 +8,7 @@ const caps = require('ssb-caps')
 const classic = require('ssb-classic/format')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-add')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-add')
 rimraf.sync(DIR)
 
 test('add() classic', async (t) => {

test/create.test.js

@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-create');
+const DIR = path.join(os.tmpdir(), 'ppppp-db-create');
 rimraf.sync(DIR)
 
 let ssb

test/del.test.js

@@ -9,7 +9,7 @@ const push = require('push-stream')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-del')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-del')
 rimraf.sync(DIR)
 
 test('del', async (t) => {

test/feed-v1-encode.test.js

@@ -0,0 +1,91 @@
+const tape = require('tape')
+const dagfeed = require('../lib/feed-v1')
+const { generateKeypair } = require('./util')
+
+tape('encode/decode works', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+  const content = { text: 'Hello world!' }
+  const timestamp = 1652037377204
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content,
+    type: 'post',
+    previous: [],
+    timestamp,
+    hmacKey,
+  })
+  t.equals(
+    nmsg1.metadata.author,
+    '4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW',
+    'metadata.author is correct'
+  )
+  t.equals(nmsg1.metadata.type, 'post', 'metadata.type is correct')
+  t.deepEquals(nmsg1.metadata.previous, [], 'metadata.previous is correct')
+  console.log(nmsg1)
+
+  const jsonMsg = {
+    key: dagfeed.getMsgId(nmsg1),
+    value: dagfeed.fromNativeMsg(nmsg1),
+    timestamp: Date.now(),
+  }
+
+  const msgHash1 = 'HEzse89DSDWUXVPyav35GC'
+  const msgKey1 =
+    'ssb:message/dag/4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW/post/' +
+    msgHash1
+
+  t.deepEqual(jsonMsg.key, msgKey1, 'key is correct')
+  t.deepEqual(
+    jsonMsg.value.author,
+    'ssb:feed/dag/4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW/post',
+    'author is correct'
+  )
+  t.deepEqual(jsonMsg.value.type, 'post', 'correct type')
+  t.equals(typeof jsonMsg.value.timestamp, 'number', 'has timestamp')
+  t.deepEqual(jsonMsg.value.previous, [], 'correct previous')
+  t.deepEqual(jsonMsg.value.content, content, 'content is the same')
+
+  const reconstructedNMsg1 = dagfeed.toNativeMsg(jsonMsg.value)
+  t.deepEqual(reconstructedNMsg1, nmsg1, 'can reconstruct')
+
+  const content2 = { text: 'Hello butty world!' }
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: content2,
+    type: 'post',
+    previous: [msgHash1],
+    timestamp: timestamp + 1,
+    hmacKey,
+  })
+  console.log(nmsg2)
+
+  const jsonMsg2 = {
+    key: dagfeed.getMsgId(nmsg2),
+    value: dagfeed.fromNativeMsg(nmsg2),
+    timestamp: Date.now(),
+  }
+
+  t.deepEqual(
+    jsonMsg2.key,
+    'ssb:message/dag/4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW/post/U5n4v1m7gFzrtrdK84gGsV',
+    'key is correct'
+  )
+  t.deepEqual(
+    jsonMsg2.value.author,
+    'ssb:feed/dag/4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW/post',
+    'author is correct'
+  )
+  t.deepEqual(jsonMsg2.value.type, 'post', 'correct type')
+  t.equals(typeof jsonMsg2.value.timestamp, 'number', 'has timestamp')
+  t.deepEqual(jsonMsg2.value.previous, [msgKey1], 'correct previous')
+  t.deepEqual(jsonMsg2.value.content, content2, 'content is the same')
+
+  // test slow version as well
+  const reconstructedNMsg2 = dagfeed.toNativeMsg(jsonMsg2.value)
+  t.deepEqual(reconstructedNMsg2, nmsg2, 'can reconstruct')
+
+  t.end()
+})

test/feed-v1-invalid-previous.test.js

@@ -0,0 +1,241 @@
+const tape = require('tape')
+const base58 = require('bs58')
+const dagfeed = require('../lib/feed-v1')
+const { generateKeypair } = require('./util')
+
+tape('invalid 1st msg with non-empty previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const fakeMsgKey0 = base58.encode(Buffer.alloc(16).fill(42))
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey0],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg1, [], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous must be an empty array/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid 1st msg with non-array previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  nmsg1.metadata.previous = null
+
+  dagfeed.validate(nmsg1, [], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous must be an array/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid msg with non-array previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = `ssb:message/dag/${base58.encode(
+    Buffer.alloc(16).fill(42)
+  )}`
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+  nmsg2.metadata.previous = null
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous must be an array/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid msg with bad previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = `ssb:message/dag/${base58.encode(
+    Buffer.alloc(16).fill(42)
+  )}`
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+  nmsg2.metadata.previous = [1234]
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous must contain strings/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid msg with SSB URI previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = `ssb:message/dag/${base58.encode(
+    Buffer.alloc(16).fill(42)
+  )}`
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+  nmsg2.metadata.previous = [fakeMsgKey1]
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous must not contain SSB URIs/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid msg with unknown previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = base58.encode(Buffer.alloc(16).fill(42))
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous .+ is not a known message ID/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})
+
+tape('invalid msg with unknown previous in a Set', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = base58.encode(Buffer.alloc(16).fill(42))
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  const existing = new Set([nmsg1])
+
+  dagfeed.validate(nmsg2, existing, null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous .+ is not a known message ID/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})

test/feed-v1-invalid-type.test.js

@@ -0,0 +1,109 @@
+const tape = require('tape')
+const dagfeed = require('../lib/feed-v1')
+const { generateKeypair } = require('./util')
+
+tape('invalid type not a string', function (t) {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  t.throws(
+    () => {
+      dagfeed.newNativeMsg({
+        keys,
+        content: { text: 'Hello world!' },
+        timestamp: 1652037377204,
+        type: 123,
+        previous: [],
+        hmacKey,
+      })
+    },
+    /type is not a string/,
+    'invalid type if contains /'
+  )
+  t.end()
+})
+
+tape('invalid type with "/" character', function (t) {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  t.throws(
+    () => {
+      dagfeed.newNativeMsg({
+        keys,
+        content: { text: 'Hello world!' },
+        timestamp: 1652037377204,
+        type: 'group/init',
+        previous: [],
+        hmacKey,
+      })
+    },
+    /invalid type/,
+    'invalid type if contains /'
+  )
+  t.end()
+})
+
+tape('invalid type with "*" character', function (t) {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  t.throws(
+    () => {
+      dagfeed.newNativeMsg({
+        keys,
+        content: { text: 'Hello world!' },
+        timestamp: 1652037377204,
+        type: 'star*',
+        previous: [],
+        hmacKey,
+      })
+    },
+    /invalid type/,
+    'invalid type if contains *'
+  )
+  t.end()
+})
+
+tape('invalid type too short', function (t) {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  t.throws(
+    () => {
+      dagfeed.newNativeMsg({
+        keys,
+        content: { text: 'Hello world!' },
+        timestamp: 1652037377204,
+        type: 'xy',
+        previous: [],
+        hmacKey,
+      })
+    },
+    /shorter than 3/,
+    'invalid type if too short'
+  )
+  t.end()
+})
+
+tape('invalid type too long', function (t) {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  t.throws(
+    () => {
+      dagfeed.newNativeMsg({
+        keys,
+        content: { text: 'Hello world!' },
+        timestamp: 1652037377204,
+        type: 'a'.repeat(120),
+        previous: [],
+        hmacKey,
+      })
+    },
+    /100\+ characters long/,
+    'invalid type if too long'
+  )
+
+  t.end()
+})

test/feed-v1-validate.test.js

@@ -0,0 +1,224 @@
+const tape = require('tape')
+const base58 = require('bs58')
+const dagfeed = require('../lib/feed-v1')
+const { generateKeypair } = require('./util')
+
+tape('validate 1st msg', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg1, null, null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 1st msg')
+    t.end()
+  })
+})
+
+tape('validate 2nd msg with existing nativeMsg', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  const msgKey1 = dagfeed.getMsgId(nmsg1)
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [msgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 2nd msg')
+    t.end()
+  })
+})
+
+tape('validate 2nd msg with existing msgId', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  const msgKey1 = dagfeed.getMsgId(nmsg1)
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [msgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2, [msgKey1], null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 2nd msg')
+    t.end()
+  })
+})
+
+tape('validate 2nd msg with existing msgId in a Set', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  const msgId1 = dagfeed.getMsgHash(nmsg1)
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [msgId1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  const existing = new Set([msgId1])
+
+  dagfeed.validate(nmsg2, existing, null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 2nd msg')
+    t.end()
+  })
+})
+
+tape('validate 2nd msg with existing KVT', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  const kvt1 = {
+    key: dagfeed.getMsgId(nmsg1),
+    value: dagfeed.fromNativeMsg(nmsg1),
+    timestamp: Date.now(),
+  }
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [kvt1.key],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2, [kvt1], null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 2nd msg')
+    t.end()
+  })
+})
+
+tape('validate 2nd forked msg', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+  const msgKey1 = dagfeed.getMsgId(nmsg1)
+
+  const nmsg2A = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [msgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  const nmsg2B = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [msgKey1],
+    timestamp: 1652030003000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2B, [nmsg1, nmsg2A], null, (err) => {
+    if (err) console.log(err)
+    t.error(err, 'valid 2nd forked msg')
+    t.end()
+  })
+})
+
+tape('invalid msg with unknown previous', (t) => {
+  const keys = generateKeypair('alice')
+  const hmacKey = null
+
+  const nmsg1 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [],
+    timestamp: 1652030001000,
+    hmacKey,
+  })
+
+  const fakeMsgKey1 = base58.encode(Buffer.alloc(16).fill(42))
+
+  const nmsg2 = dagfeed.newNativeMsg({
+    keys,
+    content: { text: 'Hello world!' },
+    type: 'post',
+    previous: [fakeMsgKey1],
+    timestamp: 1652030002000,
+    hmacKey,
+  })
+
+  dagfeed.validate(nmsg2, [nmsg1], null, (err) => {
+    t.ok(err, 'invalid 2nd msg throws')
+    t.match(
+      err.message,
+      /previous .+ is not a known message ID/,
+      'invalid 2nd msg description'
+    )
+    t.end()
+  })
+})

test/filter-as-array.test.js

@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-filter-as-array')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-filter-as-array')
 rimraf.sync(DIR)
 
 test('filterAsArray', async (t) => {

test/filter-as-iterator.test.js

@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-filter-as-iterator')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-filter-as-iterator')
 rimraf.sync(DIR)
 
 test('filterAsIterator', async (t) => {

test/filter-as-pull-stream.test.js

@@ -8,7 +8,7 @@ const caps = require('ssb-caps')
 const pull = require('pull-stream')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-filter-as-pull-stream')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-filter-as-pull-stream')
 rimraf.sync(DIR)
 
 test('filterAsPullStream', async (t) => {

test/for-each.test.js

@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-for-each')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-for-each')
 rimraf.sync(DIR)
 
 test('forEach', async (t) => {

test/on-msg-added.test.js

@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const caps = require('ssb-caps')
 const p = require('util').promisify
 
-const DIR = path.join(os.tmpdir(), 'ssb-memdb-on-msg-added')
+const DIR = path.join(os.tmpdir(), 'ppppp-db-on-msg-added')
 rimraf.sync(DIR)
 
 test('onMsgAdded', async (t) => {

test/util.js

@@ -0,0 +1,14 @@
+const ssbKeys = require('ssb-keys')
+const SSBURI = require('ssb-uri2')
+const base58 = require('bs58')
+
+function generateKeypair(seed) {
+  const keys = ssbKeys.generate('ed25519', seed, 'buttwoo-v1')
+  const { data } = SSBURI.decompose(keys.id)
+  keys.id = `ssb:feed/dag/${base58.encode(Buffer.from(data, 'base64'))}`
+  return keys
+}
+
+module.exports = {
+  generateKeypair,
+}