pzp
/
pzp-dict
mirror of https://codeberg.org/pzp/pzp-dict.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

init

This commit is contained in:
Andre Staltz 2023-04-28 14:11:16 +03:00
commit eb902078f1
No known key found for this signature in database
GPG Key ID: 9EDE23EA7E8A4890
10 changed files with 682 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/node.js.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: CI
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
matrix:
node-version: [16.x, 18.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm test

9
.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
.vscode
node_modules
pnpm-lock.yaml
package-lock.json
coverage
*~
# For misc scripts and experiments:
/gitignored

2
.prettierrc.yaml Normal file
View File

@ -0,0 +1,2 @@
semi: false
singleQuote: true

20
LICENSE Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2023 Andre 'Staltz' Medeiros <contact@staltz.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

9
README.md Normal file
View File

@ -0,0 +1,9 @@
**Work in progress**
## Installation
We're not on npm yet. In your package.json, include this as
```js
"ppppp-record": "github:staltz/ppppp-record"
```

273
lib/index.js Normal file
View File

@ -0,0 +1,273 @@
const FeedV1 = require('ppppp-db/feed-v1')
const PREFIX = 'record_v1__'
/** @typedef {string} Subtype */
/** @typedef {string} MsgHash */
/** @typedef {`${Subtype}.${string}`} SubtypeField */
/**
* @param {string} type
* @returns {Subtype}
*/
function toSubtype(type) {
return type.slice(PREFIX.length)
}
/**
* @param {Subtype} subtype
* @returns {string}
*/
function fromSubtype(subtype) {
return PREFIX + subtype
}
module.exports = {
name: 'record',
manifest: {
update: 'async',
get: 'sync',
squeeze: 'async',
},
init(peer, config) {
//#region state
const myWho = FeedV1.stripAuthor(config.keys.id)
let cancelListeningToRecordAdded = null
/** @type {Map<Subtype, unknown>} */
const tangles = new Map()
const fieldRoots = {
/** @type {Map<SubtypeField, Set<MsgHash>} */
_map: new Map(),
_getKey(subtype, field) {
return subtype + '.' + field
},
get(subtype, field = null) {
if (field) {
const key = this._getKey(subtype, field)
return this._map.get(key)
} else {
const out = {}
for (const [key, value] of this._map.entries()) {
if (key.startsWith(subtype + '.')) {
const field = key.slice(subtype.length + 1)
out[field] = [...value]
}
}
return out
}
},
add(subtype, field, msgHash) {
const key = this._getKey(subtype, field)
const set = this._map.get(key) ?? new Set()
set.add(msgHash)
return this._map.set(key, set)
},
del(subtype, field, msgHash) {
const key = this._getKey(subtype, field)
const set = this._map.get(key)
if (!set) return false
set.delete(msgHash)
if (set.size === 0) this._map.delete(key)
return true
},
toString() {
return this._map
},
}
//#endregion
//#region active processes
const loadPromise = new Promise((resolve, reject) => {
for (const { hash, msg } of peer.db.records()) {
maybeLearnAboutRecord(hash, msg)
}
cancelListeningToRecordAdded = peer.db.onRecordAdded(({ hash, msg }) => {
maybeLearnAboutRecord(hash, msg)
})
resolve()
})
peer.close.hook(function (fn, args) {
cancelListeningToRecordAdded()
fn.apply(this, args)
})
//#endregion
//#region internal methods
function isValidRecordRootMsg(msg) {
if (!msg) return false
if (msg.metadata.who !== myWho) return false
const type = msg.metadata.type
if (!type.startsWith(PREFIX)) return false
return FeedV1.isFeedRoot(msg, config.keys.id, type)
}
function isValidRecordMsg(msg) {
if (!msg) return false
if (!msg.content) return false
if (msg.metadata.who !== myWho) return false
if (!msg.metadata.type.startsWith(PREFIX)) return false
if (!msg.content.update) return false
if (typeof msg.content.update !== 'object') return false
if (Array.isArray(msg.content.update)) return false
if (!Array.isArray(msg.content.supersedes)) return false
return true
}
function learnRecordRoot(hash, msg) {
const { type } = msg.metadata
const subtype = toSubtype(type)
const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(hash)
tangle.add(hash, msg)
tangles.set(subtype, tangle)
}
function learnRecordUpdate(hash, msg) {
const { who, type } = msg.metadata
const rootHash = FeedV1.getFeedRootHash(who, type)
const subtype = toSubtype(type)
const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(rootHash)
tangle.add(hash, msg)
tangles.set(subtype, tangle)
for (const field in msg.content.update) {
const existing = fieldRoots.get(subtype, field)
if (!existing) {
fieldRoots.add(subtype, field, hash)
} else {
for (const existingHash of existing) {
if (tangle.precedes(existingHash, hash)) {
fieldRoots.del(subtype, field, existingHash)
fieldRoots.add(subtype, field, hash)
} else {
fieldRoots.add(subtype, field, hash)
}
}
}
}
}
function maybeLearnAboutRecord(hash, msg) {
if (msg.metadata.who !== myWho) return
if (isValidRecordRootMsg(msg)) {
learnRecordRoot(hash, msg)
return
}
if (isValidRecordMsg(msg)) {
learnRecordUpdate(hash, msg)
return
}
}
function loaded(cb) {
if (cb === void 0) return loadPromise
else loadPromise.then(() => cb(null), cb)
}
function _getFieldRoots(subtype) {
return fieldRoots.get(subtype)
}
function _squeezePotential(subtype) {
const rootHash = FeedV1.getFeedRootHash(myWho, fromSubtype(subtype))
const tangle = peer.db.getTangle(rootHash)
const maxDepth = tangle.getMaxDepth()
const fieldRoots = _getFieldRoots(subtype)
let minDepth = Infinity
for (const field in fieldRoots) {
for (const msgHash of fieldRoots[field]) {
const depth = tangle.getDepth(msgHash)
if (depth < minDepth) minDepth = depth
}
}
return maxDepth - minDepth
}
function forceUpdate(subtype, update, cb) {
const type = fromSubtype(subtype)
// Populate supersedes
const supersedes = []
for (const field in update) {
const existing = fieldRoots.get(subtype, field)
if (existing) supersedes.push(...existing)
}
peer.db.create({ type, content: { update, supersedes } }, (err, rec) => {
// prettier-ignore
if (err) return cb(new Error('Failed to create msg when force updating Record', { cause: err }))
cb(null, true)
})
}
//#endregion
//#region public methods
function get(authorId, subtype) {
const type = fromSubtype(subtype)
const rootHash = FeedV1.getFeedRootHash(authorId, type)
const tangle = peer.db.getTangle(rootHash)
if (!tangle || tangle.size() === 0) return {}
const msgHashes = tangle.topoSort()
const record = {}
for (const msgHash of msgHashes) {
const msg = peer.db.get(msgHash)
if (isValidRecordMsg(msg)) {
const { update } = msg.content
Object.assign(record, update)
}
}
return record
}
function update(authorId, subtype, update, cb) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error('Cannot update another user\'s record. Given "authorId" was ' + authorId))
loaded(() => {
const record = get(authorId, subtype)
let hasChanges = false
for (const [field, value] of Object.entries(update)) {
if (value !== record[field]) {
hasChanges = true
break
}
}
if (!hasChanges) return cb(null, false)
forceUpdate(subtype, update, cb)
})
}
function squeeze(authorId, subtype, cb) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error('Cannot squeeze another user\'s record. Given "authorId" was ' + authorId))
const potential = _squeezePotential(subtype)
if (potential < 1) return cb(null, false)
loaded(() => {
const record = get(authorId, subtype)
forceUpdate(subtype, record, (err) => {
// prettier-ignore
if (err) return cb(new Error('Failed to force update when squeezing Record', { cause: err }))
cb(null, true)
})
})
}
//#endregion
return {
update,
get,
squeeze,
_getFieldRoots,
_squeezePotential,
}
},
}

53
package.json Normal file
View File

@ -0,0 +1,53 @@
{
"name": "ppppp-record",
"version": "1.0.0",
"description": "Record data structure over append-only logs with pruning",
"author": "Andre Staltz <contact@staltz.com>",
"license": "MIT",
"homepage": "https://github.com/staltz/ppppp-record",
"repository": {
"type": "git",
"url": "git@github.com:staltz/ppppp-record.git"
},
"main": "index.js",
"files": [
"*.js",
"lib/*.js"
],
"exports": {
".": {
"require": "./lib/index.js"
}
},
"type": "commonjs",
"engines": {
"node": ">=16"
},
"dependencies": {
},
"devDependencies": {
"bs58": "^5.0.0",
"c8": "7",
"ppppp-db": "github:staltz/ppppp-db",
"rimraf": "^4.4.0",
"secret-stack": "^6.4.1",
"ssb-box": "^1.0.1",
"ssb-caps": "^1.1.0",
"ssb-classic": "^1.1.0",
"ssb-keys": "^8.5.0",
"ssb-uri2": "^2.4.1",
"tap-arc": "^0.3.5",
"tape": "^5.6.3"
},
"scripts": {
"test": "tape test/*.js | tap-arc --bail",
"format-code": "prettier --write \"(lib|test)/**/*.js\"",
"format-code-staged": "pretty-quick --staged --pattern \"(lib|test)/**/*.js\"",
"coverage": "c8 --reporter=lcov npm run test"
},
"husky": {
"hooks": {
"pre-commit": "npm run format-code-staged"
}
}
}

132
protospec.md Normal file
View File

@ -0,0 +1,132 @@
## Feed tangle
(Lipmaa backlinks are not shown in the diagram below, but they should exist)
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]
B[updates name]
C[updates age]
D[updates name]
E[updates age & name]
C-->B-->A-->R
D--->A
E-->D & C
```
Reducing the tangle above in a topological sort allows you to build a record
(a JSON object) `{age, name}`.
## Msg type
`msg.metadata.type` MUST start with `record_v1__`. E.g. `record_v1__profile`.
## Msg content
`msg.content` format:
```typescript
interface MsgContent {
update: Record<string, any>,
supersedes: Array<MsgHash>,
}
```
RECOMMENDED that the `msg.content.update` is as flat as possible (no nesting).
## Supersedes links
When you update a field in a record, in the `supersedes` array you MUST point
to the currently-known highest-depth msg that updated that field.
The set of *not-transitively-superseded-by-anyone* msgs comprise the
"field roots" of the record. To allow pruning the tangle, we can delete
(or, if we want to keep metadata, "erase") all msgs preceding the field roots.
Suppose the tangle is grown in the order below, then the field roots are
highlighted in blue.
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]:::blue
A-->R
classDef blue fill:#69f,stroke:#fff0,color:#000
```
----
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]:::blue
B[updates name]:::blue
B-->A-->R
classDef blue fill:#69f,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]
B[updates name]:::blue
C[updates age]:::blue
C-->B-->A-->R
C-- supersedes -->A
linkStyle 3 stroke-width:1px,stroke:#05f
classDef blue fill:#69f,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]
B[updates name]:::blue
C[updates age]:::blue
D[updates name]:::blue
C-->B-->A-->R
D--->A
C-- supersedes -->A
linkStyle 4 stroke-width:1px,stroke:#05f
classDef blue fill:#69f,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]:::feedroot
A[updates age]
B[updates name]
C[updates age]
D[updates name]
E[updates age & name]:::blue
C-->B-->A-->R
C-- supersedes -->A
D-->A
E-->D & C
E-- supersedes -->C
E-- supersedes -->D
linkStyle 3,7,8 stroke-width:1px,stroke:#05f
classDef blue fill:#69f,stroke:#fff0,color:#000
```

145
test/index.test.js Normal file
View File

@ -0,0 +1,145 @@
const test = require('tape')
const path = require('path')
const os = require('os')
const rimraf = require('rimraf')
const SecretStack = require('secret-stack')
const FeedV1 = require('ppppp-db/feed-v1')
const caps = require('ssb-caps')
const p = require('util').promisify
const { generateKeypair } = require('./util')
const DIR = path.join(os.tmpdir(), 'ppppp-record')
rimraf.sync(DIR)
const aliceKeys = generateKeypair('alice')
const who = aliceKeys.id
let peer
test('setup', async (t) => {
peer = SecretStack({ appKey: caps.shs })
.use(require('ppppp-db'))
.use(require('ssb-box'))
.use(require('../lib'))
.call(null, {
keys: aliceKeys,
path: DIR,
})
await peer.db.loaded()
})
test('Record update() and get()', async (t) => {
t.ok(
await p(peer.record.update)(who, 'profile', { name: 'alice' }),
'update .name'
)
t.deepEqual(peer.record.get(who, 'profile'), { name: 'alice' }, 'get')
const fieldRoots1 = peer.record._getFieldRoots('profile')
t.deepEquals(fieldRoots1, { name: ['Pt4YwxksvCLir45Tmw3hXK'] }, 'fieldRoots')
t.ok(await p(peer.record.update)(who, 'profile', { age: 20 }), 'update .age')
t.deepEqual(
peer.record.get(who, 'profile'),
{ name: 'alice', age: 20 },
'get'
)
const fieldRoots2 = peer.record._getFieldRoots('profile')
t.deepEquals(
fieldRoots2,
{ name: ['Pt4YwxksvCLir45Tmw3hXK'], age: ['XqkG9Uz1eQcxv9R1f3jgKS'] },
'fieldRoots'
)
t.false(
await p(peer.record.update)(who, 'profile', { name: 'alice' }),
'redundant update .name'
)
t.deepEqual(
peer.record.get(who, 'profile'),
{ name: 'alice', age: 20 },
'get'
)
t.true(
await p(peer.record.update)(who, 'profile', { name: 'Alice' }),
'update .name'
)
t.deepEqual(
peer.record.get(who, 'profile'),
{ name: 'Alice', age: 20 },
'get'
)
const fieldRoots3 = peer.record._getFieldRoots('profile')
t.deepEquals(
fieldRoots3,
{ name: ['WGDGt1UEGPpRyutfDyC2we'], age: ['XqkG9Uz1eQcxv9R1f3jgKS'] },
'fieldRoots'
)
})
test('Record squeeze', async (t) => {
t.ok(await p(peer.record.update)(who, 'profile', { age: 21 }), 'update .age')
t.ok(await p(peer.record.update)(who, 'profile', { age: 22 }), 'update .age')
t.ok(await p(peer.record.update)(who, 'profile', { age: 23 }), 'update .age')
const fieldRoots4 = peer.record._getFieldRoots('profile')
t.deepEquals(
fieldRoots4,
{ name: ['WGDGt1UEGPpRyutfDyC2we'], age: ['6qu5mbLbFPJHCFge7QtU48'] },
'fieldRoots'
)
t.equals(peer.record._squeezePotential('profile'), 3, 'squeezePotential=3')
t.true(await p(peer.record.squeeze)(who, 'profile'), 'squeezed')
const fieldRoots5 = peer.record._getFieldRoots('profile')
t.deepEquals(
fieldRoots5,
{ name: ['Ba96TjutuuPbdMMvNS4BbL'], age: ['Ba96TjutuuPbdMMvNS4BbL'] },
'fieldRoots'
)
t.equals(peer.record._squeezePotential('profile'), 0, 'squeezePotential=0')
t.false(await p(peer.record.squeeze)(who, 'profile'), 'squeeze idempotent')
const fieldRoots6 = peer.record._getFieldRoots('profile')
t.deepEquals(fieldRoots6, fieldRoots5, 'fieldRoots')
})
test('Record receives old branched update', async (t) => {
const rootMsg = FeedV1.createRoot(aliceKeys, 'record_v1__profile')
const rootHash = FeedV1.getMsgHash(rootMsg)
const tangle = new FeedV1.Tangle(rootHash)
tangle.add(rootHash, rootMsg)
const msg = FeedV1.create({
keys: aliceKeys,
type: 'record_v1__profile',
content: { update: { age: 2 }, supersedes: [] },
tangles: {
[rootHash]: tangle,
},
})
const rec = await p(peer.db.add)(msg, rootHash)
t.equals(rec.hash, 'JXvFSXE9s1DF77cSu5XUm', 'msg hash')
const fieldRoots7 = peer.record._getFieldRoots('profile')
t.deepEquals(
fieldRoots7,
{
name: ['Ba96TjutuuPbdMMvNS4BbL'],
age: ['Ba96TjutuuPbdMMvNS4BbL', rec.hash],
},
'fieldRoots'
)
t.equals(peer.record._squeezePotential('profile'), 6, 'squeezePotential=6')
})
test('teardown', (t) => {
peer.close(true, t.end)
})

14
test/util.js Normal file
View File

@ -0,0 +1,14 @@
const ssbKeys = require('ssb-keys')
const SSBURI = require('ssb-uri2')
const base58 = require('bs58')
function generateKeypair(seed) {
const keys = ssbKeys.generate('ed25519', seed, 'buttwoo-v1')
const { data } = SSBURI.decompose(keys.id)
keys.id = `ppppp:feed/v1/${base58.encode(Buffer.from(data, 'base64'))}`
return keys
}
module.exports = {
generateKeypair,
}