commit b1fb6c3eea7f919a3a2a33bd6cb67d6a829ec1bd Author: Andre Staltz Date: Fri May 5 16:27:52 2023 +0300 init diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml new file mode 100644 index 0000000..d6ab510 --- /dev/null +++ b/.github/workflows/node.js.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + matrix: + node-version: [16.x, 18.x] + + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - run: npm install + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b96477 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.vscode +node_modules +pnpm-lock.yaml +package-lock.json +coverage +*~ + +# For misc scripts and experiments: +/gitignored diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000..1d2127c --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,2 @@ +semi: false +singleQuote: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fcb6945 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2023 Andre 'Staltz' Medeiros + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4aa7f19 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +**Work in progress** + +## Installation + +We're not on npm yet. In your package.json, include this as + +```js +"ppppp-set": "github:staltz/ppppp-set" +``` diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..a3b3d1b --- /dev/null +++ b/lib/index.js @@ -0,0 +1,331 @@ +const FeedV1 = require('ppppp-db/feed-v1') + +const PREFIX = 'set_v1__' + +/** @typedef {string} Subtype */ + +/** @typedef {string} MsgHash */ + +/** @typedef {`${Subtype}.${string}`} SubtypeItem */ + +/** + * @param {string} type + * @returns {Subtype} + */ +function toSubtype(type) { + return type.slice(PREFIX.length) +} + +/** + * @param {Subtype} subtype + * @returns {string} + */ +function fromSubtype(subtype) { + return PREFIX + subtype +} + +module.exports = { + name: 'set', + manifest: { + add: 'async', + del: 'async', + has: 'sync', + values: 'sync', + getItemRoots: 'sync', + squeeze: 'async', + }, + init(peer, config) { + //#region state + const myWho = FeedV1.stripAuthor(config.keys.id) + let cancelListeningToRecordAdded = null + + /** @type {Map} */ + const tangles = new Map() + + const itemRoots = { + /** @type {Map} */ + _map: new Map(), + _getKey(subtype, item) { + return subtype + '/' + item + }, + get(subtype, item = null) { + if (item) { + const key = this._getKey(subtype, item) + return this._map.get(key) + } else { + const out = {} + for (const [key, value] of this._map.entries()) { + if (key.startsWith(subtype + '/')) { + const item = key.slice(subtype.length + 1) + out[item] = [...value] + } + } + return out + } + }, + add(subtype, item, msgHash) { + const key = this._getKey(subtype, item) + const set = this._map.get(key) ?? new Set() + set.add(msgHash) + return this._map.set(key, set) + }, + del(subtype, item, msgHash) { + const key = this._getKey(subtype, item) + const set = this._map.get(key) + if (!set) return false + set.delete(msgHash) + if (set.size === 0) this._map.delete(key) + return true + }, + toString() { + return this._map + }, + } + //#endregion + + //#region active processes + const loadPromise = new Promise((resolve, reject) => { + for (const { hash, msg } of peer.db.records()) { + maybeLearnAboutSet(hash, msg) + } + cancelListeningToRecordAdded = peer.db.onRecordAdded(({ hash, msg }) => { + maybeLearnAboutSet(hash, msg) + }) + resolve() + }) + + peer.close.hook(function (fn, args) { + cancelListeningToRecordAdded() + fn.apply(this, args) + }) + //#endregion + + //#region internal methods + function isValidSetRootMsg(msg) { + if (!msg) return false + if (msg.metadata.who !== myWho) return false + const type = msg.metadata.type + if (!type.startsWith(PREFIX)) return false + return FeedV1.isFeedRoot(msg, config.keys.id, type) + } + + function isValidSetMsg(msg) { + if (!msg) return false + if (!msg.content) return false + if (msg.metadata.who !== myWho) return false + if (!msg.metadata.type.startsWith(PREFIX)) return false + if (!Array.isArray(msg.content.add)) return false + if (!Array.isArray(msg.content.del)) return false + if (!Array.isArray(msg.content.supersedes)) return false + return true + } + + function readSet(authorId, subtype) { + const type = fromSubtype(subtype) + const rootHash = FeedV1.getFeedRootHash(authorId, type) + const tangle = peer.db.getTangle(rootHash) + if (!tangle || tangle.size() === 0) return new Set() + const msgHashes = tangle.topoSort() + const set = new Set() + for (const msgHash of msgHashes) { + const msg = peer.db.get(msgHash) + if (isValidSetMsg(msg)) { + const { add, del } = msg.content + for (const value of add) set.add(value) + for (const value of del) set.delete(value) + } + } + return set + } + + function learnSetRoot(hash, msg) { + const { type } = msg.metadata + const subtype = toSubtype(type) + const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(hash) + tangle.add(hash, msg) + tangles.set(subtype, tangle) + } + + function learnSetUpdate(hash, msg) { + const { who, type } = msg.metadata + const rootHash = FeedV1.getFeedRootHash(who, type) + const subtype = toSubtype(type) + const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(rootHash) + tangle.add(hash, msg) + tangles.set(subtype, tangle) + const addOrRemove = [].concat(msg.content.add, msg.content.del) + for (const item of addOrRemove) { + const existing = itemRoots.get(subtype, item) + if (!existing || existing.size === 0) { + itemRoots.add(subtype, item, hash) + } else { + for (const existingHash of existing) { + if (tangle.precedes(existingHash, hash)) { + itemRoots.del(subtype, item, existingHash) + itemRoots.add(subtype, item, hash) + } else { + itemRoots.add(subtype, item, hash) + } + } + } + } + } + + function maybeLearnAboutSet(hash, msg) { + if (msg.metadata.who !== myWho) return + if (isValidSetRootMsg(msg)) { + learnSetRoot(hash, msg) + return + } + if (isValidSetMsg(msg)) { + learnSetUpdate(hash, msg) + return + } + } + + function loaded(cb) { + if (cb === void 0) return loadPromise + else loadPromise.then(() => cb(null), cb) + } + + function _squeezePotential(subtype) { + // TODO: improve this so that the squeezePotential is the size of the + // tangle suffix built as a slice from the fieldRoots + const rootHash = FeedV1.getFeedRootHash(myWho, fromSubtype(subtype)) + const tangle = peer.db.getTangle(rootHash) + const maxDepth = tangle.getMaxDepth() + const currentItemRoots = itemRoots.get(subtype) + let minDepth = Infinity + for (const item in currentItemRoots) { + for (const msgHash of currentItemRoots[item]) { + const depth = tangle.getDepth(msgHash) + if (depth < minDepth) minDepth = depth + } + } + return maxDepth - minDepth + } + //#endregion + + //#region public methods + function add(authorId, subtype, value, cb) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error(`Cannot add to another user's Set (${authorId}/${subtype})`)) + + loaded(() => { + const currentSet = readSet(authorId, subtype) + if (currentSet.has(value)) return cb(null, false) + const type = fromSubtype(subtype) + + // Populate supersedes + const supersedes = [] + const toDeleteFromItemRoots = new Map() + const currentItemRoots = itemRoots.get(subtype) + for (const item in currentItemRoots) { + // If we are re-adding this item, OR if this item has been deleted, + // then we should update roots + if (item === value || !currentSet.has(item)) { + supersedes.push(...currentItemRoots[item]) + for (const msgHash of currentItemRoots[item]) { + toDeleteFromItemRoots.set(msgHash, item) + } + } + } + + const content = { add: [value], del: [], supersedes } + peer.db.create({ type, content }, (err) => { + // prettier-ignore + if (err) return cb(new Error(`Failed to create msg when adding to Set (${authorId}/${subtype})`, { cause: err })) + for (const [msgHash, item] of toDeleteFromItemRoots) { + itemRoots.del(subtype, item, msgHash) + } + cb(null, true) + }) + }) + } + + function del(authorId, subtype, value, cb) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error(`Cannot delete from another user's Set (${authorId}/${subtype})`)) + + loaded(() => { + const currentSet = readSet(authorId, subtype) + if (!currentSet.has(value)) return cb(null, false) + const type = fromSubtype(subtype) + + // Populate supersedes + const supersedes = [] + const currentItemRoots = itemRoots.get(subtype) + for (const item in currentItemRoots) { + if (item === value || !currentSet.has(item)) { + supersedes.push(...currentItemRoots[item]) + } + } + + const content = { add: [], del: [value], supersedes } + peer.db.create({ type, content }, (err) => { + // prettier-ignore + if (err) return cb(new Error(`Failed to create msg when deleting from Set (${authorId}/${subtype})`, { cause: err })) + cb(null, true) + }) + }) + } + + function has(authorId, subtype, value) { + const set = readSet(authorId, subtype) + return set.has(value) + } + + function values(authorId, subtype) { + const set = readSet(authorId, subtype) + return [...set] + } + + function getItemRoots(authorId, subtype) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error(`Cannot getItemRoots of another user's Set. (${authorId}/${subtype})`)) + return itemRoots.get(subtype) + } + + function squeeze(authorId, subtype, cb) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error(`Cannot squeeze another user's Set (${authorId}/${subtype})`)) + + const potential = _squeezePotential(subtype) + if (potential < 1) return cb(null, false) + + loaded(() => { + const type = fromSubtype(subtype) + const currentSet = readSet(authorId, subtype) + + const supersedes = [] + const currentItemRoots = itemRoots.get(subtype) + for (const item in currentItemRoots) { + supersedes.push(...currentItemRoots[item]) + } + + const content = { add: [...currentSet], del: [], supersedes } + peer.db.create({ type, content }, (err) => { + // prettier-ignore + if (err) return cb(new Error(`Failed to create msg when squeezing Set (${authorId}/${subtype})`, { cause: err })) + cb(null, true) + }) + }) + } + //#endregion + + return { + add, + del, + has, + values, + getItemRoots, + squeeze, + + _squeezePotential, + } + }, +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..05cf919 --- /dev/null +++ b/package.json @@ -0,0 +1,53 @@ +{ + "name": "ppppp-set", + "version": "1.0.0", + "description": "Set data structure over append-only logs with pruning", + "author": "Andre Staltz ", + "license": "MIT", + "homepage": "https://github.com/staltz/ppppp-set", + "repository": { + "type": "git", + "url": "git@github.com:staltz/ppppp-set.git" + }, + "main": "index.js", + "files": [ + "*.js", + "lib/*.js" + ], + "exports": { + ".": { + "require": "./lib/index.js" + } + }, + "type": "commonjs", + "engines": { + "node": ">=16" + }, + "dependencies": { + }, + "devDependencies": { + "bs58": "^5.0.0", + "c8": "7", + "ppppp-db": "github:staltz/ppppp-db", + "rimraf": "^4.4.0", + "secret-stack": "^6.4.1", + "ssb-box": "^1.0.1", + "ssb-caps": "^1.1.0", + "ssb-classic": "^1.1.0", + "ssb-keys": "^8.5.0", + "ssb-uri2": "^2.4.1", + "tap-arc": "^0.3.5", + "tape": "^5.6.3" + }, + "scripts": { + "test": "tape test/*.js | tap-arc --bail", + "format-code": "prettier --write \"(lib|test)/**/*.js\"", + "format-code-staged": "pretty-quick --staged --pattern \"(lib|test)/**/*.js\"", + "coverage": "c8 --reporter=lcov npm run test" + }, + "husky": { + "hooks": { + "pre-commit": "npm run format-code-staged" + } + } +} diff --git a/protospec.md b/protospec.md new file mode 100644 index 0000000..a1d0907 --- /dev/null +++ b/protospec.md @@ -0,0 +1,139 @@ +## Feed tangle + +(Lipmaa backlinks are not shown in the diagram below, but they should exist) + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice] +B[adds bob] +C[deletes alices] +D[adds bob] +E[adds carol] + +C-->B-->A-->R +D--->A +E-->D & C +classDef default fill:#bbb,stroke:#fff0,color:#000 +``` + +Reducing the tangle above in a topological sort allows you to build an array +(a JSON object) `[bob, carol]`. + +## Msg type + +`msg.metadata.type` MUST start with `set_v1__`. E.g. `set_v1__follows`. + +## Msg content + +`msg.content` format: + +```typescript +interface MsgContent { + add: Array, + del: Array, + supersedes: Array, +} +``` + +## Supersedes links + +When you add or delete an item in a set, in the `supersedes` array you MUST +point to the currently-known highest-depth msg that added or deleted that item. + +Also, when you *add any item*, in the `supersedes` array you SHOULD point to +all currently-known highest-depth msgs that *deleted something*. + +The set of *not-transitively-superseded-by-anyone* msgs comprise the +"item roots" of the record. To allow pruning the tangle, we can delete +(or, if we want to keep metadata, "erase") all msgs preceding the item roots. + +Suppose the tangle is grown in the order below, then the field roots are +highlighted in blue. + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice]:::blue + +A-->R +classDef default fill:#bbb,stroke:#fff0,color:#000 +classDef blue fill:#6af,stroke:#fff0,color:#000 +``` + +----- + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice]:::blue +B[adds bob]:::blue + +B-->A-->R +classDef default fill:#bbb,stroke:#fff0,color:#000 +classDef blue fill:#6af,stroke:#fff0,color:#000 +``` + +----- + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice] +B[adds bob]:::blue +C[deletes alices]:::blue + +C-->B-->A-->R +C-- supersedes -->A + +linkStyle 3 stroke-width:1px,stroke:#05f +classDef default fill:#bbb,stroke:#fff0,color:#000 +classDef blue fill:#6af,stroke:#fff0,color:#000 +``` + +----- + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice] +B[adds bob]:::blue +C[deletes alices]:::blue +D[adds bob]:::blue + +C-->B-->A-->R +C-- supersedes -->A +D--->A + +linkStyle 3 stroke-width:1px,stroke:#05f +classDef default fill:#bbb,stroke:#fff0,color:#000 +classDef blue fill:#6af,stroke:#fff0,color:#000 +``` + +----- + +```mermaid +graph RL + +R["(Feed root)"] +A[adds alice] +B[adds bob]:::blue +C[deletes alices] +D[adds bob]:::blue +E[adds carol]:::blue + +C-->B-->A-->R +C-- supersedes -->A +D--->A +E-->D & C +E-- supersedes -->C + +linkStyle 3,7 stroke-width:1px,stroke:#05f +classDef default fill:#bbb,stroke:#fff0,color:#000 +classDef blue fill:#6af,stroke:#fff0,color:#000 +``` diff --git a/test/index.test.js b/test/index.test.js new file mode 100644 index 0000000..7b4b7cf --- /dev/null +++ b/test/index.test.js @@ -0,0 +1,148 @@ +const test = require('tape') +const path = require('path') +const os = require('os') +const rimraf = require('rimraf') +const SecretStack = require('secret-stack') +const FeedV1 = require('ppppp-db/feed-v1') +const caps = require('ssb-caps') +const p = require('util').promisify +const { generateKeypair } = require('./util') + +const DIR = path.join(os.tmpdir(), 'ppppp-set') +rimraf.sync(DIR) + +const aliceKeys = generateKeypair('alice') +const who = aliceKeys.id + +let peer +test('setup', async (t) => { + peer = SecretStack({ appKey: caps.shs }) + .use(require('ppppp-db')) + .use(require('ssb-box')) + .use(require('../lib')) + .call(null, { + keys: aliceKeys, + path: DIR, + }) + + await peer.db.loaded() +}) + +function lastMsgHash() { + let last + for (const item of peer.db.records()) { + last = item + } + return last.hash +} + +let add1, add2, del1, add3, del2 +test('Set add(), del(), has()', async (t) => { + // Add 1st + t.false(peer.set.has(who, 'follows', '1st'), 'doesnt have 1st') + t.ok(await p(peer.set.add)(who, 'follows', '1st'), 'add 1st') + t.true(peer.set.has(who, 'follows', '1st'), 'has 1st') + add1 = lastMsgHash() + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '1st': [add1] }, + 'itemRoots' + ) + + // Add 2nd + t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd') + t.ok(await p(peer.set.add)(who, 'follows', '2nd'), 'add 2nd') + t.true(peer.set.has(who, 'follows', '2nd'), 'has 2nd') + add2 = lastMsgHash() + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '1st': [add1], '2nd': [add2] }, + 'itemRoots' + ) + + // Del 1st + t.true(peer.set.has(who, 'follows', '1st'), 'has 1st') + t.ok(await p(peer.set.del)(who, 'follows', '1st'), 'del 1st') + t.false(peer.set.has(who, 'follows', '1st'), 'doesnt have 1st') + del1 = lastMsgHash() + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '1st': [del1], '2nd': [add2] }, + 'itemRoots' + ) + + // Add 3rd + t.false(peer.set.has(who, 'follows', '3rd'), 'doesnt have 3rd') + t.ok(await p(peer.set.add)(who, 'follows', '3rd'), 'add 3rd') + t.true(peer.set.has(who, 'follows', '3rd'), 'has 3rd') + add3 = lastMsgHash() + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '3rd': [add3], '2nd': [add2] }, + 'itemRoots' + ) + + // Del 2nd + t.true(peer.set.has(who, 'follows', '2nd'), 'has 2nd') + t.ok(await p(peer.set.del)(who, 'follows', '2nd'), 'del 2nd') // msg seq 4 + t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd') + del2 = lastMsgHash() + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '3rd': [add3], '2nd': [del2] }, + 'itemRoots' + ) + + // Del 2nd (idempotent) + t.notOk(await p(peer.set.del)(who, 'follows', '2nd'), 'del 2nd idempotent') + t.false(peer.set.has(who, 'follows', '2nd'), 'doesnt have 2nd') + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '3rd': [add3], '2nd': [del2] }, + 'itemRoots' + ) +}) + +let add4, add5 +test('Set values()', async (t) => { + t.ok(await p(peer.set.add)(who, 'follows', '4th'), 'add 4th') + add4 = lastMsgHash() + t.ok(await p(peer.set.add)(who, 'follows', '5th'), 'add 5th') + add5 = lastMsgHash() + + const expected = new Set(['3rd', '4th', '5th']) + for (const item of peer.set.values(who, 'follows')) { + t.true(expected.has(item), 'values() item') + expected.delete(item) + } + t.equals(expected.size, 0, 'all items') +}) + +test('predsl Set squeeze', async (t) => { + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '3rd': [add3], '4th': [add4], '5th': [add5] }, + 'itemRoots before squeeze' + ) + + t.equals(peer.set._squeezePotential('follows'), 3, 'squeezePotential=3') + + t.true(await p(peer.set.squeeze)(who, 'follows'), 'squeezed') + const squeezed = lastMsgHash() + + t.equals(peer.set._squeezePotential('follows'), 0, 'squeezePotential=0') + + t.deepEquals( + peer.set.getItemRoots(who, 'follows'), + { '3rd': [squeezed], '4th': [squeezed], '5th': [squeezed] }, + 'itemRoots after squeeze' + ) + + t.false(await p(peer.set.squeeze)(who, 'follows'), 'squeeze again idempotent') + const squeezed2 = lastMsgHash() + t.equals(squeezed, squeezed2, 'squeezed msg hash is same') +}) + +test('teardown', (t) => { + peer.close(t.end) +}) diff --git a/test/util.js b/test/util.js new file mode 100644 index 0000000..51feb19 --- /dev/null +++ b/test/util.js @@ -0,0 +1,14 @@ +const ssbKeys = require('ssb-keys') +const SSBURI = require('ssb-uri2') +const base58 = require('bs58') + +function generateKeypair(seed) { + const keys = ssbKeys.generate('ed25519', seed, 'buttwoo-v1') + const { data } = SSBURI.decompose(keys.id) + keys.id = `ppppp:feed/v1/${base58.encode(Buffer.from(data, 'base64'))}` + return keys +} + +module.exports = { + generateKeypair, +}