pzp
/
pzp-db
mirror of https://codeberg.org/pzp/pzp-db.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add lipmaa prev

This commit is contained in:
Andre Staltz 2023-04-04 13:44:33 +03:00
parent 83d941e2dc
commit 1f53758fe0
10 changed files with 266 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/encryption.js
View File

@ -17,8 +17,7 @@ function ciphertextStrToBuffer(str) {
*/
function decrypt(rec, peer, config) {
const msgEncrypted = rec.msg
const { metadata, content } = msgEncrypted
const { who, prev } = metadata
const { content } = msgEncrypted
if (typeof content !== 'string') return rec
const encryptionFormat = peer.db.findEncryptionFormatFor(content)
@ -26,7 +25,7 @@ function decrypt(rec, peer, config) {
// Decrypt
const ciphertextBuf = ciphertextStrToBuffer(content)
const opts = { keys: config.keys, author: who, previous: prev }
const opts = { keys: config.keys }
const plaintextBuf = encryptionFormat.decrypt(ciphertextBuf, opts)
if (!plaintextBuf) return rec

89
lib/feed-v1/index.js
View File

@ -5,7 +5,7 @@
const stringify = require('fast-json-stable-stringify')
const ed25519 = require('ssb-keys/sodium')
const base58 = require('bs58')
const { stripAuthor, stripMsgKey } = require('./strip')
const { stripAuthor } = require('./strip')
const { getMsgId, getMsgHash } = require('./get-msg-id')
const representContent = require('./represent-content')
const {
@ -31,6 +31,18 @@ const {
* @property {string} sig
*/
/**
* @typedef {Object} CreateOpts
* @property {*} content
* @property {string} type
* @property {number} when
* @property {Object} keys
* @property {string} keys.id
* @property {string} keys.private
* @property {Iterator<Msg> & {values: () => Iterator<Msg>}} existing
* @property {Iterator<Msg> & {values: () => Iterator<Msg>}} tips
*/
/**
* @param {Msg} msg
*/
@ -62,47 +74,90 @@ function toPlaintextBuffer(opts) {
return Buffer.from(stringify(opts.content), 'utf8')
}
function calculateDepth(prev) {
let max = -1;
for (const p of prev) {
function calculateDepth(tips) {
let max = -1
for (const p of tips.values()) {
if (p.metadata.depth > max) {
max = p.metadata.depth;
max = p.metadata.depth
}
}
return max + 1
}
function summarizePrev(prev) {
return Array.from(prev).map(getMsgHash)
function lipmaa(n) {
let m = 1
let po3 = 3
let u = n
// find k such that (3^k - 1)/2 >= n
while (m < n) {
po3 *= 3
m = (po3 - 1) / 2
}
// find longest possible backjump
po3 /= 3
if (m !== n) {
while (u !== 0) {
m = (po3 - 1) / 2
po3 /= 3
u %= m
}
if (m !== po3) {
po3 = m
}
}
return n - po3
}
function prevalidatePrev(prev) {
if (prev && !prev[Symbol.iterator]) {
// prettier-ignore
throw new Error('opts.prev must be an iterator, but got ' + typeof prev)
function calculatePrev(existing, depth, lipmaaDepth) {
const prev = []
for (const msg of existing.values()) {
const msgDepth = msg.metadata.depth
if (msgDepth === lipmaaDepth || msgDepth === depth - 1) {
prev.push(getMsgHash(msg))
}
for (const p of prev) {
}
return prev
}
function prevalidatePrevious(prev, name) {
if (!prev?.[Symbol.iterator]) {
// prettier-ignore
throw new Error(`opts.${name} must be an iterator, but got ${typeof prev}`)
}
if (typeof prev?.values !== 'function') {
// prettier-ignore
throw new Error(`opts.${name} must be a Map, Set, or Array, but got ${prev}`)
}
for (const p of prev.values()) {
if (!p.metadata) {
throw new Error('opts.prev must contain messages, but got ' + typeof p)
throw new Error(`opts.${name} must contain messages, but got ${typeof p}`)
}
}
}
/**
* @param {*} opts
* @param {CreateOpts} opts
* @returns {Msg}
*/
function create(opts) {
let err
if ((err = validateType(opts.type))) throw err
prevalidatePrev(opts.prev)
prevalidatePrevious(opts.existing, 'existing')
prevalidatePrevious(opts.tips, 'tips')
const [proof, size] = representContent(opts.content)
const depth = calculateDepth(opts.prev)
const depth = calculateDepth(opts.tips)
const lipmaaDepth = lipmaa(depth + 1) - 1
const prev = calculatePrev(opts.existing, depth, lipmaaDepth)
const msg = {
content: opts.content,
metadata: {
depth,
prev: summarizePrev(opts.prev),
prev,
proof,
size,
type: opts.type,

48
lib/plugin.js
View File

@ -49,8 +49,8 @@ exports.init = function initDB(peer, config) {
const onRecordAdded = Obz()
const msgsPerFeed = {
_mapAll: new Map(), // who => Set<Msg>
_mapTips: new Map(), // who => Set<Msg>
_mapAll: new Map(), // who => Set<MsgHash>
_mapTips: new Map(), // who => Set<MsgHash>
_byHash: new Map(), // msgId => Msg // TODO: optimize space usage of this??
update(msg, msgId) {
const msgHash = FeedV1.getMsgHash(msgId ?? msg)
@ -58,28 +58,37 @@ exports.init = function initDB(peer, config) {
const setAll = this._mapAll.get(feedId) ?? new Set()
const setTips = this._mapTips.get(feedId) ?? new Set()
for (const p of msg.metadata.prev) {
const prevMsg = this._byHash.get(p)
setTips.delete(prevMsg)
setTips.delete(p)
}
setAll.add(msg)
setTips.add(msg)
setAll.add(msgHash)
setTips.add(msgHash)
this._mapTips.set(feedId, setTips)
this._mapAll.set(feedId, setAll)
this._byHash.set(msgHash, msg)
},
getAll() {
return this._byHash
getAll(feedId) {
const map = new Map()
for (const msgHash of this._mapAll.get(feedId) ?? []) {
const msg = this._byHash.get(msgHash)
if (msg) map.set(msgHash, msg)
}
return map
},
getTips(feedId) {
return this._mapTips.get(feedId) ?? []
const map = new Map()
for (const msgHash of this._mapTips.get(feedId) ?? []) {
const msg = this._byHash.get(msgHash)
if (msg) map.set(msgHash, msg)
}
return map
},
deleteMsg(msg) {
const feedId = FeedV1.getFeedId(msg)
const msgHash = FeedV1.getMsgHash(msg)
const setAll = this._mapAll.get(feedId)
setAll.delete(msg)
setAll.delete(msgHash)
const setTips = this._mapTips.get(feedId)
setTips.delete(msg)
setTips.delete(msgHash)
this._byHash.delete(msgHash)
},
}
@ -187,6 +196,8 @@ exports.init = function initDB(peer, config) {
function add(msg, cb) {
const feedId = FeedV1.getFeedId(msg)
// TODO: optimize this. This may be slow if you're adding many msgs in a
// row, because `getAll()` creates a new Map() each time.
const existingMsgs = msgsPerFeed.getAll(feedId)
FeedV1.validate(msg, existingMsgs, validationCB)
@ -223,19 +234,26 @@ exports.init = function initDB(peer, config) {
// Create full opts:
let tempMsg
try {
tempMsg = FeedV1.create({ when: Date.now(), ...opts, prev: [], keys })
tempMsg = FeedV1.create({
when: Date.now(),
...opts,
existing: [],
tips: [],
keys,
})
} catch (err) {
return cb(new Error('create() failed', { cause: err }))
}
const feedId = FeedV1.getFeedId(tempMsg)
const prev = msgsPerFeed.getTips(feedId)
const fullOpts = { when: Date.now(), ...opts, prev, keys }
const existing = msgsPerFeed.getAll(feedId)
const tips = msgsPerFeed.getTips(feedId)
const fullOpts = { when: Date.now(), ...opts, existing, tips, keys }
// If opts ask for encryption, encrypt and put ciphertext in opts.content
const recps = fullOpts.content.recps
if (Array.isArray(recps) && recps.length > 0) {
const plaintext = FeedV1.toPlaintextBuffer(fullOpts)
const encryptOpts = { ...fullOpts, keys, recps, prev }
const encryptOpts = { ...fullOpts, recps }
let ciphertextBuf
try {
ciphertextBuf = encryptionFormat.encrypt(plaintext, encryptOpts)

3
test/add.test.js
View File

@ -25,7 +25,8 @@ test('add()', async (t) => {
when: 1514517067954,
type: 'post',
content: { text: 'This is the first post!' },
prev: [],
existing: [],
tips: [],
})
const rec = await p(peer.db.add)(inputMsg)

11
test/create.test.js
View File

@ -48,7 +48,8 @@ test('add() forked then create() merged', async (t) => {
when: Date.now(),
type: 'post',
content: { text: '3rd post forked from 1st' },
prev: [rec1.msg],
existing: [rec1.msg],
tips: [rec1.msg],
})
const rec3 = await p(peer.db.add)(msg3)
@ -64,14 +65,6 @@ test('add() forked then create() merged', async (t) => {
[msgHash2, msgHash3],
'msg4 prev is msg2 and msg3'
)
const msgHash4 = FeedV1.getMsgHash(rec4.msg)
const rec5 = await p(peer.db.create)({
type: 'post',
content: { text: 'I am 5th post' },
})
t.ok(rec5, '5th post created')
t.deepEquals(rec5.msg.metadata.prev, [msgHash4], 'msg5 prev is msg4')
})
test('create() encrypted with box', async (t) => {

12
test/feed-v1-encode.test.js
View File

@ -11,7 +11,8 @@ tape('encode/decode works', (t) => {
keys,
content,
type: 'post',
prev: [],
existing: [],
tips: [],
when,
})
t.deepEquals(
@ -34,12 +35,12 @@ tape('encode/decode works', (t) => {
console.log(msg1)
const msgHash = '9cYegpVpddoMSdvSf53dTH'
const msgHash1 = '9cYegpVpddoMSdvSf53dTH'
t.equals(
FeedV1.getMsgId(msg1),
'ppppp:message/v1/4mjQ5aJu378cEu6TksRG3uXAiKFiwGjYQtWAjfVjDAJW/post/' +
msgHash,
msgHash1,
'getMsgId'
)
@ -49,7 +50,8 @@ tape('encode/decode works', (t) => {
keys,
content: content2,
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: when + 1,
})
t.deepEquals(
@ -64,7 +66,7 @@ tape('encode/decode works', (t) => {
)
t.equals(msg2.metadata.type, 'post', 'metadata.type')
t.equals(msg2.metadata.depth, 1, 'metadata.depth')
t.deepEquals(msg2.metadata.prev, [msgHash], 'metadata.prev')
t.deepEquals(msg2.metadata.prev, [msgHash1], 'metadata.prev')
t.deepEquals(msg2.metadata.proof, 'XuZEzH1Dhy1yuRMcviBBcN', 'metadata.proof')
t.deepEquals(msg2.metadata.size, 21, 'metadata.size')
t.equals(typeof msg2.metadata.when, 'number', 'metadata.when')

70
test/feed-v1-invalid-prev.test.js
View File

@ -10,7 +10,8 @@ tape('invalid 1st msg with non-empty prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [{ metadata: { depth: 10 }, sig: 'fake' }],
existing: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
tips: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
when: 1652030001000,
})
@ -32,7 +33,8 @@ tape('invalid 1st msg with non-array prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
msg.metadata.prev = null
@ -51,7 +53,8 @@ tape('invalid msg with non-array prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
@ -60,7 +63,8 @@ tape('invalid msg with non-array prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [{ metadata: { depth: 10 }, sig: 'fake' }],
existing: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
tips: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
when: 1652030002000,
})
msg2.metadata.prev = null
@ -85,16 +89,18 @@ tape('invalid msg with bad prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
const msgHash1 = FeedV1.getMsgHash(msg1)
const msg2 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [{ metadata: { depth: 10 }, sig: 'fake' }],
existing: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
tips: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
when: 1652030002000,
})
msg2.metadata.prev = [1234]
@ -119,7 +125,8 @@ tape('invalid msg with URI in prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
@ -128,7 +135,8 @@ tape('invalid msg with URI in prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [{ metadata: { depth: 10 }, sig: 'fake' }],
existing: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
tips: new Map([['1234', { metadata: { depth: 10 }, sig: 'fake' }]]),
when: 1652030002000,
})
const randBuf = Buffer.alloc(16).fill(16)
@ -155,7 +163,8 @@ tape('invalid msg with unknown prev', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
@ -164,15 +173,18 @@ tape('invalid msg with unknown prev', (t) => {
keys,
content: { text: 'Alien' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const unknownMsgHash = FeedV1.getMsgHash(unknownMsg)
const msg2 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [unknownMsg],
existing: new Map([[unknownMsgHash, unknownMsg]]),
tips: new Map([[unknownMsgHash, unknownMsg]]),
when: 1652030002000,
})
@ -188,37 +200,3 @@ tape('invalid msg with unknown prev', (t) => {
t.end()
})
})
tape('invalid msg with unknown prev in a Set', (t) => {
const keys = generateKeypair('alice')
const msg1 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
when: 1652030001000,
})
const msg2 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [{ metadata: { depth: 10 }, sig: 'fake' }],
when: 1652030002000,
})
const fakeMsgKey1 = base58.encode(Buffer.alloc(16).fill(42))
msg2.metadata.prev = [fakeMsgKey1]
const existing = new Set([msg1])
FeedV1.validate(msg2, existing, (err) => {
t.ok(err, 'invalid 2nd msg throws')
t.match(
err.message,
/prev .+ is not locally known/,
'invalid 2nd msg description'
)
t.end()
})
})

15
test/feed-v1-invalid-type.test.js
View File

@ -12,7 +12,8 @@ tape('invalid type not a string', (t) => {
content: { text: 'Hello world!' },
when: 1652037377204,
type: 123,
prev: [],
existing: new Map(),
tips: new Map(),
})
},
/type is not a string/,
@ -31,7 +32,8 @@ tape('invalid type with "/" character', (t) => {
content: { text: 'Hello world!' },
when: 1652037377204,
type: 'group/init',
prev: [],
existing: new Map(),
tips: new Map(),
})
},
/invalid type/,
@ -50,7 +52,8 @@ tape('invalid type with "*" character', (t) => {
content: { text: 'Hello world!' },
when: 1652037377204,
type: 'star*',
prev: [],
existing: new Map(),
tips: new Map(),
})
},
/invalid type/,
@ -69,7 +72,8 @@ tape('invalid type too short', (t) => {
content: { text: 'Hello world!' },
when: 1652037377204,
type: 'xy',
prev: [],
existing: new Map(),
tips: new Map(),
})
},
/shorter than 3/,
@ -88,7 +92,8 @@ tape('invalid type too long', (t) => {
content: { text: 'Hello world!' },
when: 1652037377204,
type: 'a'.repeat(120),
prev: [],
existing: new Map(),
tips: new Map(),
})
},
/100\+ characters long/,

86
test/feed-v1-lipmaa.test.js Normal file
View File

@ -0,0 +1,86 @@
const tape = require('tape')
const FeedV1 = require('../lib/feed-v1')
const { generateKeypair } = require('./util')
tape('lipmaa prevs', (t) => {
const keys = generateKeypair('alice')
const content = { text: 'Hello world!' }
const when = 1652037377204
const existing = new Map()
const tips = new Map()
const msg1 = FeedV1.create({
keys,
content,
type: 'post',
existing: new Map(),
tips: new Map(),
when: when + 1,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
existing.set(msgHash1, msg1)
tips.set(msgHash1, msg1)
t.deepEquals(msg1.metadata.prev, [], 'msg1.prev is empty')
const msg2 = FeedV1.create({
keys,
content,
type: 'post',
existing,
tips,
when: when + 2,
})
const msgHash2 = FeedV1.getMsgHash(msg2)
existing.set(msgHash2, msg2)
tips.set(msgHash2, msg2)
tips.delete(msgHash1)
t.deepEquals(msg2.metadata.prev, [msgHash1], 'msg2.prev is msg1')
const msg3 = FeedV1.create({
keys,
content,
type: 'post',
existing,
tips,
when: when + 3,
})
const msgHash3 = FeedV1.getMsgHash(msg3)
existing.set(msgHash3, msg3)
tips.set(msgHash3, msg3)
tips.delete(msgHash2)
t.deepEquals(msg3.metadata.prev, [msgHash2], 'msg3.prev is msg2')
const msg4 = FeedV1.create({
keys,
content,
type: 'post',
existing,
tips,
when: when + 4,
})
const msgHash4 = FeedV1.getMsgHash(msg4)
existing.set(msgHash4, msg4)
tips.set(msgHash4, msg4)
tips.delete(msgHash3)
t.deepEquals(
msg4.metadata.prev,
[msgHash1, msgHash3],
'msg4.prev is msg1 and msg3'
)
const msg5 = FeedV1.create({
keys,
content,
type: 'post',
existing,
tips,
when: when + 5,
})
const msgHash5 = FeedV1.getMsgHash(msg5)
existing.set(msgHash5, msg5)
tips.set(msgHash5, msg5)
tips.delete(msgHash4)
t.deepEquals(msg5.metadata.prev, [msgHash4], 'msg5.prev is msg4')
t.end()
})

44
test/feed-v1-validate.test.js
View File

@ -10,7 +10,8 @@ tape('validate 1st msg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
@ -28,7 +29,8 @@ tape('validate 2nd msg with existing nativeMsg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
@ -37,7 +39,8 @@ tape('validate 2nd msg with existing nativeMsg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030002000,
})
@ -58,16 +61,18 @@ tape('validate 2nd msg with existing msgId', (t) => {
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgKey1 = FeedV1.getMsgId(msg1)
const msgHash1 = FeedV1.getMsgHash(msg1)
const msgHash1 = FeedV1.getMsgHash(msg1)
const msg2 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030002000,
})
@ -87,22 +92,24 @@ tape('validate 2nd msg with existing KVT', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
const msgHash1 = FeedV1.getMsgHash(msg1)
const msg2 = FeedV1.create({
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030002000,
})
const existing = new Map()
existing.set(msgHash1, msg1)
FeedV1.validate(msg2,existing, (err) => {
FeedV1.validate(msg2, existing, (err) => {
if (err) console.log(err)
t.error(err, 'valid 2nd msg')
t.end()
@ -116,7 +123,8 @@ tape('validate 2nd forked msg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
@ -125,7 +133,8 @@ tape('validate 2nd forked msg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030002000,
})
const msgHash2A = FeedV1.getMsgHash(msg2A)
@ -134,7 +143,8 @@ tape('validate 2nd forked msg', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030003000,
})
@ -155,10 +165,11 @@ tape('invalid msg with unknown previous', (t) => {
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [],
existing: new Map(),
tips: new Map(),
when: 1652030001000,
})
const msgHash1 = FeedV1.getMsgHash(msg1)
const msgHash1 = FeedV1.getMsgHash(msg1)
const fakeMsgKey1 = base58.encode(Buffer.alloc(16).fill(42))
@ -166,7 +177,8 @@ const msgHash1 = FeedV1.getMsgHash(msg1)
keys,
content: { text: 'Hello world!' },
type: 'post',
prev: [msg1],
existing: new Map([[msgHash1, msg1]]),
tips: new Map([[msgHash1, msg1]]),
when: 1652030002000,
})
msg2.metadata.prev = [fakeMsgKey1]