This commit is contained in:
Andre Staltz 2023-05-05 16:27:52 +03:00
commit b1fb6c3eea
No known key found for this signature in database
GPG Key ID: 9EDE23EA7E8A4890
10 changed files with 750 additions and 0 deletions

25
.github/workflows/node.js.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: CI
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
matrix:
node-version: [16.x, 18.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm test

9
.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
.vscode
node_modules
pnpm-lock.yaml
package-lock.json
coverage
*~
# For misc scripts and experiments:
/gitignored

2
.prettierrc.yaml Normal file
View File

@ -0,0 +1,2 @@
semi: false
singleQuote: true

20
LICENSE Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2023 Andre 'Staltz' Medeiros <contact@staltz.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

9
README.md Normal file
View File

@ -0,0 +1,9 @@
**Work in progress**
## Installation
We're not on npm yet. In your package.json, include this as
```js
"ppppp-set": "github:staltz/ppppp-set"
```

331
lib/index.js Normal file
View File

@ -0,0 +1,331 @@
const FeedV1 = require('ppppp-db/feed-v1')
const PREFIX = 'set_v1__'
/** @typedef {string} Subtype */
/** @typedef {string} MsgHash */
/** @typedef {`${Subtype}.${string}`} SubtypeItem */
/**
* @param {string} type
* @returns {Subtype}
*/
function toSubtype(type) {
return type.slice(PREFIX.length)
}
/**
* @param {Subtype} subtype
* @returns {string}
*/
function fromSubtype(subtype) {
return PREFIX + subtype
}
module.exports = {
name: 'set',
manifest: {
add: 'async',
del: 'async',
has: 'sync',
values: 'sync',
getItemRoots: 'sync',
squeeze: 'async',
},
init(peer, config) {
//#region state
const myWho = FeedV1.stripAuthor(config.keys.id)
let cancelListeningToRecordAdded = null
/** @type {Map<Subtype, unknown>} */
const tangles = new Map()
const itemRoots = {
/** @type {Map<SubtypeItem, Set<MsgHash>} */
_map: new Map(),
_getKey(subtype, item) {
return subtype + '/' + item
},
get(subtype, item = null) {
if (item) {
const key = this._getKey(subtype, item)
return this._map.get(key)
} else {
const out = {}
for (const [key, value] of this._map.entries()) {
if (key.startsWith(subtype + '/')) {
const item = key.slice(subtype.length + 1)
out[item] = [...value]
}
}
return out
}
},
add(subtype, item, msgHash) {
const key = this._getKey(subtype, item)
const set = this._map.get(key) ?? new Set()
set.add(msgHash)
return this._map.set(key, set)
},
del(subtype, item, msgHash) {
const key = this._getKey(subtype, item)
const set = this._map.get(key)
if (!set) return false
set.delete(msgHash)
if (set.size === 0) this._map.delete(key)
return true
},
toString() {
return this._map
},
}
//#endregion
//#region active processes
const loadPromise = new Promise((resolve, reject) => {
for (const { hash, msg } of peer.db.records()) {
maybeLearnAboutSet(hash, msg)
}
cancelListeningToRecordAdded = peer.db.onRecordAdded(({ hash, msg }) => {
maybeLearnAboutSet(hash, msg)
})
resolve()
})
peer.close.hook(function (fn, args) {
cancelListeningToRecordAdded()
fn.apply(this, args)
})
//#endregion
//#region internal methods
function isValidSetRootMsg(msg) {
if (!msg) return false
if (msg.metadata.who !== myWho) return false
const type = msg.metadata.type
if (!type.startsWith(PREFIX)) return false
return FeedV1.isFeedRoot(msg, config.keys.id, type)
}
function isValidSetMsg(msg) {
if (!msg) return false
if (!msg.content) return false
if (msg.metadata.who !== myWho) return false
if (!msg.metadata.type.startsWith(PREFIX)) return false
if (!Array.isArray(msg.content.add)) return false
if (!Array.isArray(msg.content.del)) return false
if (!Array.isArray(msg.content.supersedes)) return false
return true
}
function readSet(authorId, subtype) {
const type = fromSubtype(subtype)
const rootHash = FeedV1.getFeedRootHash(authorId, type)
const tangle = peer.db.getTangle(rootHash)
if (!tangle || tangle.size() === 0) return new Set()
const msgHashes = tangle.topoSort()
const set = new Set()
for (const msgHash of msgHashes) {
const msg = peer.db.get(msgHash)
if (isValidSetMsg(msg)) {
const { add, del } = msg.content
for (const value of add) set.add(value)
for (const value of del) set.delete(value)
}
}
return set
}
function learnSetRoot(hash, msg) {
const { type } = msg.metadata
const subtype = toSubtype(type)
const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(hash)
tangle.add(hash, msg)
tangles.set(subtype, tangle)
}
function learnSetUpdate(hash, msg) {
const { who, type } = msg.metadata
const rootHash = FeedV1.getFeedRootHash(who, type)
const subtype = toSubtype(type)
const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(rootHash)
tangle.add(hash, msg)
tangles.set(subtype, tangle)
const addOrRemove = [].concat(msg.content.add, msg.content.del)
for (const item of addOrRemove) {
const existing = itemRoots.get(subtype, item)
if (!existing || existing.size === 0) {
itemRoots.add(subtype, item, hash)
} else {
for (const existingHash of existing) {
if (tangle.precedes(existingHash, hash)) {
itemRoots.del(subtype, item, existingHash)
itemRoots.add(subtype, item, hash)
} else {
itemRoots.add(subtype, item, hash)
}
}
}
}
}
function maybeLearnAboutSet(hash, msg) {
if (msg.metadata.who !== myWho) return
if (isValidSetRootMsg(msg)) {
learnSetRoot(hash, msg)
return
}
if (isValidSetMsg(msg)) {
learnSetUpdate(hash, msg)
return
}
}
function loaded(cb) {
if (cb === void 0) return loadPromise
else loadPromise.then(() => cb(null), cb)
}
function _squeezePotential(subtype) {
// TODO: improve this so that the squeezePotential is the size of the
// tangle suffix built as a slice from the fieldRoots
const rootHash = FeedV1.getFeedRootHash(myWho, fromSubtype(subtype))
const tangle = peer.db.getTangle(rootHash)
const maxDepth = tangle.getMaxDepth()
const currentItemRoots = itemRoots.get(subtype)
let minDepth = Infinity
for (const item in currentItemRoots) {
for (const msgHash of currentItemRoots[item]) {
const depth = tangle.getDepth(msgHash)
if (depth < minDepth) minDepth = depth
}
}
return maxDepth - minDepth
}
//#endregion
//#region public methods
function add(authorId, subtype, value, cb) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error(`Cannot add to another user's Set (${authorId}/${subtype})`))
loaded(() => {
const currentSet = readSet(authorId, subtype)
if (currentSet.has(value)) return cb(null, false)
const type = fromSubtype(subtype)
// Populate supersedes
const supersedes = []
const toDeleteFromItemRoots = new Map()
const currentItemRoots = itemRoots.get(subtype)
for (const item in currentItemRoots) {
// If we are re-adding this item, OR if this item has been deleted,
// then we should update roots
if (item === value || !currentSet.has(item)) {
supersedes.push(...currentItemRoots[item])
for (const msgHash of currentItemRoots[item]) {
toDeleteFromItemRoots.set(msgHash, item)
}
}
}
const content = { add: [value], del: [], supersedes }
peer.db.create({ type, content }, (err) => {
// prettier-ignore
if (err) return cb(new Error(`Failed to create msg when adding to Set (${authorId}/${subtype})`, { cause: err }))
for (const [msgHash, item] of toDeleteFromItemRoots) {
itemRoots.del(subtype, item, msgHash)
}
cb(null, true)
})
})
}
function del(authorId, subtype, value, cb) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error(`Cannot delete from another user's Set (${authorId}/${subtype})`))
loaded(() => {
const currentSet = readSet(authorId, subtype)
if (!currentSet.has(value)) return cb(null, false)
const type = fromSubtype(subtype)
// Populate supersedes
const supersedes = []
const currentItemRoots = itemRoots.get(subtype)
for (const item in currentItemRoots) {
if (item === value || !currentSet.has(item)) {
supersedes.push(...currentItemRoots[item])
}
}
const content = { add: [], del: [value], supersedes }
peer.db.create({ type, content }, (err) => {
// prettier-ignore
if (err) return cb(new Error(`Failed to create msg when deleting from Set (${authorId}/${subtype})`, { cause: err }))
cb(null, true)
})
})
}
function has(authorId, subtype, value) {
const set = readSet(authorId, subtype)
return set.has(value)
}
function values(authorId, subtype) {
const set = readSet(authorId, subtype)
return [...set]
}
function getItemRoots(authorId, subtype) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error(`Cannot getItemRoots of another user's Set. (${authorId}/${subtype})`))
return itemRoots.get(subtype)
}
function squeeze(authorId, subtype, cb) {
const who = FeedV1.stripAuthor(authorId)
// prettier-ignore
if (who !== myWho) return cb(new Error(`Cannot squeeze another user's Set (${authorId}/${subtype})`))
const potential = _squeezePotential(subtype)
if (potential < 1) return cb(null, false)
loaded(() => {
const type = fromSubtype(subtype)
const currentSet = readSet(authorId, subtype)
const supersedes = []
const currentItemRoots = itemRoots.get(subtype)
for (const item in currentItemRoots) {
supersedes.push(...currentItemRoots[item])
}
const content = { add: [...currentSet], del: [], supersedes }
peer.db.create({ type, content }, (err) => {
// prettier-ignore
if (err) return cb(new Error(`Failed to create msg when squeezing Set (${authorId}/${subtype})`, { cause: err }))
cb(null, true)
})
})
}
//#endregion
return {
add,
del,
has,
values,
getItemRoots,
squeeze,
_squeezePotential,
}
},
}

53
package.json Normal file
View File

@ -0,0 +1,53 @@
{
"name": "ppppp-set",
"version": "1.0.0",
"description": "Set data structure over append-only logs with pruning",
"author": "Andre Staltz <contact@staltz.com>",
"license": "MIT",
"homepage": "https://github.com/staltz/ppppp-set",
"repository": {
"type": "git",
"url": "git@github.com:staltz/ppppp-set.git"
},
"main": "index.js",
"files": [
"*.js",
"lib/*.js"
],
"exports": {
".": {
"require": "./lib/index.js"
}
},
"type": "commonjs",
"engines": {
"node": ">=16"
},
"dependencies": {
},
"devDependencies": {
"bs58": "^5.0.0",
"c8": "7",
"ppppp-db": "github:staltz/ppppp-db",
"rimraf": "^4.4.0",
"secret-stack": "^6.4.1",
"ssb-box": "^1.0.1",
"ssb-caps": "^1.1.0",
"ssb-classic": "^1.1.0",
"ssb-keys": "^8.5.0",
"ssb-uri2": "^2.4.1",
"tap-arc": "^0.3.5",
"tape": "^5.6.3"
},
"scripts": {
"test": "tape test/*.js | tap-arc --bail",
"format-code": "prettier --write \"(lib|test)/**/*.js\"",
"format-code-staged": "pretty-quick --staged --pattern \"(lib|test)/**/*.js\"",
"coverage": "c8 --reporter=lcov npm run test"
},
"husky": {
"hooks": {
"pre-commit": "npm run format-code-staged"
}
}
}

139
protospec.md Normal file
View File

@ -0,0 +1,139 @@
## Feed tangle
(Lipmaa backlinks are not shown in the diagram below, but they should exist)
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]
B[adds bob]
C[deletes alices]
D[adds bob]
E[adds carol]
C-->B-->A-->R
D--->A
E-->D & C
classDef default fill:#bbb,stroke:#fff0,color:#000
```
Reducing the tangle above in a topological sort allows you to build an array
(a JSON object) `[bob, carol]`.
## Msg type
`msg.metadata.type` MUST start with `set_v1__`. E.g. `set_v1__follows`.
## Msg content
`msg.content` format:
```typescript
interface MsgContent {
add: Array<string>,
del: Array<string>,
supersedes: Array<MsgHash>,
}
```
## Supersedes links
When you add or delete an item in a set, in the `supersedes` array you MUST
point to the currently-known highest-depth msg that added or deleted that item.
Also, when you *add any item*, in the `supersedes` array you SHOULD point to
all currently-known highest-depth msgs that *deleted something*.
The set of *not-transitively-superseded-by-anyone* msgs comprise the
"item roots" of the record. To allow pruning the tangle, we can delete
(or, if we want to keep metadata, "erase") all msgs preceding the item roots.
Suppose the tangle is grown in the order below, then the field roots are
highlighted in blue.
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]:::blue
A-->R
classDef default fill:#bbb,stroke:#fff0,color:#000
classDef blue fill:#6af,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]:::blue
B[adds bob]:::blue
B-->A-->R
classDef default fill:#bbb,stroke:#fff0,color:#000
classDef blue fill:#6af,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]
B[adds bob]:::blue
C[deletes alices]:::blue
C-->B-->A-->R
C-- supersedes -->A
linkStyle 3 stroke-width:1px,stroke:#05f
classDef default fill:#bbb,stroke:#fff0,color:#000
classDef blue fill:#6af,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]
B[adds bob]:::blue
C[deletes alices]:::blue
D[adds bob]:::blue
C-->B-->A-->R
C-- supersedes -->A
D--->A
linkStyle 3 stroke-width:1px,stroke:#05f
classDef default fill:#bbb,stroke:#fff0,color:#000
classDef blue fill:#6af,stroke:#fff0,color:#000
```
-----
```mermaid
graph RL
R["(Feed root)"]
A[adds alice]
B[adds bob]:::blue
C[deletes alices]
D[adds bob]:::blue
E[adds carol]:::blue
C-->B-->A-->R
C-- supersedes -->A
D--->A
E-->D & C
E-- supersedes -->C
linkStyle 3,7 stroke-width:1px,stroke:#05f
classDef default fill:#bbb,stroke:#fff0,color:#000
classDef blue fill:#6af,stroke:#fff0,color:#000
```

148
test/index.test.js Normal file
View File

@ -0,0 +1,148 @@
const test = require('tape')
const path = require('path')
const os = require('os')
const rimraf = require('rimraf')
const SecretStack = require('secret-stack')
const FeedV1 = require('ppppp-db/feed-v1')
const caps = require('ssb-caps')
const p = require('util').promisify
const { generateKeypair } = require('./util')
const DIR = path.join(os.tmpdir(), 'ppppp-set')
rimraf.sync(DIR)
const aliceKeys = generateKeypair('alice')
const who = aliceKeys.id
let peer
test('setup', async (t) => {
peer = SecretStack({ appKey: caps.shs })
.use(require('ppppp-db'))
.use(require('ssb-box'))
.use(require('../lib'))
.call(null, {
keys: aliceKeys,
path: DIR,
})
await peer.db.loaded()
})
function lastMsgHash() {
let last
for (const item of peer.db.records()) {
last = item
}
return last.hash
}
let add1, add2, del1, add3, del2
test('Set add(), del(), has()', async (t) => {
// Add 1st
t.false(peer.set.has(who, 'follows', '1st'), 'doesnt have 1st')
t.ok(await p(peer.set.add)(who, 'follows', '1st'), 'add 1st')
t.true(peer.set.has(who, 'follows', '1st'), 'has 1st')
add1 = lastMsgHash()
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '1st': [add1] },
'itemRoots'
)
// Add 2nd
t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd')
t.ok(await p(peer.set.add)(who, 'follows', '2nd'), 'add 2nd')
t.true(peer.set.has(who, 'follows', '2nd'), 'has 2nd')
add2 = lastMsgHash()
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '1st': [add1], '2nd': [add2] },
'itemRoots'
)
// Del 1st
t.true(peer.set.has(who, 'follows', '1st'), 'has 1st')
t.ok(await p(peer.set.del)(who, 'follows', '1st'), 'del 1st')
t.false(peer.set.has(who, 'follows', '1st'), 'doesnt have 1st')
del1 = lastMsgHash()
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '1st': [del1], '2nd': [add2] },
'itemRoots'
)
// Add 3rd
t.false(peer.set.has(who, 'follows', '3rd'), 'doesnt have 3rd')
t.ok(await p(peer.set.add)(who, 'follows', '3rd'), 'add 3rd')
t.true(peer.set.has(who, 'follows', '3rd'), 'has 3rd')
add3 = lastMsgHash()
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '3rd': [add3], '2nd': [add2] },
'itemRoots'
)
// Del 2nd
t.true(peer.set.has(who, 'follows', '2nd'), 'has 2nd')
t.ok(await p(peer.set.del)(who, 'follows', '2nd'), 'del 2nd') // msg seq 4
t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd')
del2 = lastMsgHash()
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '3rd': [add3], '2nd': [del2] },
'itemRoots'
)
// Del 2nd (idempotent)
t.notOk(await p(peer.set.del)(who, 'follows', '2nd'), 'del 2nd idempotent')
t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd')
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '3rd': [add3], '2nd': [del2] },
'itemRoots'
)
})
let add4, add5
test('Set values()', async (t) => {
t.ok(await p(peer.set.add)(who, 'follows', '4th'), 'add 4th')
add4 = lastMsgHash()
t.ok(await p(peer.set.add)(who, 'follows', '5th'), 'add 5th')
add5 = lastMsgHash()
const expected = new Set(['3rd', '4th', '5th'])
for (const item of peer.set.values(who, 'follows')) {
t.true(expected.has(item), 'values() item')
expected.delete(item)
}
t.equals(expected.size, 0, 'all items')
})
test('predsl Set squeeze', async (t) => {
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '3rd': [add3], '4th': [add4], '5th': [add5] },
'itemRoots before squeeze'
)
t.equals(peer.set._squeezePotential('follows'), 3, 'squeezePotential=3')
t.true(await p(peer.set.squeeze)(who, 'follows'), 'squeezed')
const squeezed = lastMsgHash()
t.equals(peer.set._squeezePotential('follows'), 0, 'squeezePotential=0')
t.deepEquals(
peer.set.getItemRoots(who, 'follows'),
{ '3rd': [squeezed], '4th': [squeezed], '5th': [squeezed] },
'itemRoots after squeeze'
)
t.false(await p(peer.set.squeeze)(who, 'follows'), 'squeeze again idempotent')
const squeezed2 = lastMsgHash()
t.equals(squeezed, squeezed2, 'squeezed msg hash is same')
})
test('teardown', (t) => {
peer.close(t.end)
})

14
test/util.js Normal file
View File

@ -0,0 +1,14 @@
const ssbKeys = require('ssb-keys')
const SSBURI = require('ssb-uri2')
const base58 = require('bs58')
function generateKeypair(seed) {
const keys = ssbKeys.generate('ed25519', seed, 'buttwoo-v1')
const { data } = SSBURI.decompose(keys.id)
keys.id = `ppppp:feed/v1/${base58.encode(Buffer.from(data, 'base64'))}`
return keys
}
module.exports = {
generateKeypair,
}