From eb902078f1eb33f695aea3334ba6d1755c631f77 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 28 Apr 2023 14:11:16 +0300 Subject: [PATCH] init --- .github/workflows/node.js.yml | 25 ++++ .gitignore | 9 ++ .prettierrc.yaml | 2 + LICENSE | 20 +++ README.md | 9 ++ lib/index.js | 273 ++++++++++++++++++++++++++++++++++ package.json | 53 +++++++ protospec.md | 132 ++++++++++++++++ test/index.test.js | 145 ++++++++++++++++++ test/util.js | 14 ++ 10 files changed, 682 insertions(+) create mode 100644 .github/workflows/node.js.yml create mode 100644 .gitignore create mode 100644 .prettierrc.yaml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 lib/index.js create mode 100644 package.json create mode 100644 protospec.md create mode 100644 test/index.test.js create mode 100644 test/util.js diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml new file mode 100644 index 0000000..d6ab510 --- /dev/null +++ b/.github/workflows/node.js.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + matrix: + node-version: [16.x, 18.x] + + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - run: npm install + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b96477 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.vscode +node_modules +pnpm-lock.yaml +package-lock.json +coverage +*~ + +# For misc scripts and experiments: +/gitignored diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000..1d2127c --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,2 @@ +semi: false +singleQuote: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fcb6945 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2023 Andre 'Staltz' Medeiros + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..def90a1 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +**Work in progress** + +## Installation + +We're not on npm yet. In your package.json, include this as + +```js +"ppppp-record": "github:staltz/ppppp-record" +``` diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..01c4b20 --- /dev/null +++ b/lib/index.js @@ -0,0 +1,273 @@ +const FeedV1 = require('ppppp-db/feed-v1') + +const PREFIX = 'record_v1__' + +/** @typedef {string} Subtype */ + +/** @typedef {string} MsgHash */ + +/** @typedef {`${Subtype}.${string}`} SubtypeField */ + +/** + * @param {string} type + * @returns {Subtype} + */ +function toSubtype(type) { + return type.slice(PREFIX.length) +} + +/** + * @param {Subtype} subtype + * @returns {string} + */ +function fromSubtype(subtype) { + return PREFIX + subtype +} + +module.exports = { + name: 'record', + manifest: { + update: 'async', + get: 'sync', + squeeze: 'async', + }, + init(peer, config) { + //#region state + const myWho = FeedV1.stripAuthor(config.keys.id) + let cancelListeningToRecordAdded = null + + /** @type {Map} */ + const tangles = new Map() + + const fieldRoots = { + /** @type {Map} */ + _map: new Map(), + _getKey(subtype, field) { + return subtype + '.' + field + }, + get(subtype, field = null) { + if (field) { + const key = this._getKey(subtype, field) + return this._map.get(key) + } else { + const out = {} + for (const [key, value] of this._map.entries()) { + if (key.startsWith(subtype + '.')) { + const field = key.slice(subtype.length + 1) + out[field] = [...value] + } + } + return out + } + }, + add(subtype, field, msgHash) { + const key = this._getKey(subtype, field) + const set = this._map.get(key) ?? new Set() + set.add(msgHash) + return this._map.set(key, set) + }, + del(subtype, field, msgHash) { + const key = this._getKey(subtype, field) + const set = this._map.get(key) + if (!set) return false + set.delete(msgHash) + if (set.size === 0) this._map.delete(key) + return true + }, + toString() { + return this._map + }, + } + //#endregion + + //#region active processes + const loadPromise = new Promise((resolve, reject) => { + for (const { hash, msg } of peer.db.records()) { + maybeLearnAboutRecord(hash, msg) + } + cancelListeningToRecordAdded = peer.db.onRecordAdded(({ hash, msg }) => { + maybeLearnAboutRecord(hash, msg) + }) + resolve() + }) + + peer.close.hook(function (fn, args) { + cancelListeningToRecordAdded() + fn.apply(this, args) + }) + //#endregion + + //#region internal methods + function isValidRecordRootMsg(msg) { + if (!msg) return false + if (msg.metadata.who !== myWho) return false + const type = msg.metadata.type + if (!type.startsWith(PREFIX)) return false + return FeedV1.isFeedRoot(msg, config.keys.id, type) + } + + function isValidRecordMsg(msg) { + if (!msg) return false + if (!msg.content) return false + if (msg.metadata.who !== myWho) return false + if (!msg.metadata.type.startsWith(PREFIX)) return false + if (!msg.content.update) return false + if (typeof msg.content.update !== 'object') return false + if (Array.isArray(msg.content.update)) return false + if (!Array.isArray(msg.content.supersedes)) return false + return true + } + + function learnRecordRoot(hash, msg) { + const { type } = msg.metadata + const subtype = toSubtype(type) + const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(hash) + tangle.add(hash, msg) + tangles.set(subtype, tangle) + } + + function learnRecordUpdate(hash, msg) { + const { who, type } = msg.metadata + const rootHash = FeedV1.getFeedRootHash(who, type) + const subtype = toSubtype(type) + const tangle = tangles.get(subtype) ?? new FeedV1.Tangle(rootHash) + tangle.add(hash, msg) + tangles.set(subtype, tangle) + + for (const field in msg.content.update) { + const existing = fieldRoots.get(subtype, field) + if (!existing) { + fieldRoots.add(subtype, field, hash) + } else { + for (const existingHash of existing) { + if (tangle.precedes(existingHash, hash)) { + fieldRoots.del(subtype, field, existingHash) + fieldRoots.add(subtype, field, hash) + } else { + fieldRoots.add(subtype, field, hash) + } + } + } + } + } + + function maybeLearnAboutRecord(hash, msg) { + if (msg.metadata.who !== myWho) return + if (isValidRecordRootMsg(msg)) { + learnRecordRoot(hash, msg) + return + } + if (isValidRecordMsg(msg)) { + learnRecordUpdate(hash, msg) + return + } + } + + function loaded(cb) { + if (cb === void 0) return loadPromise + else loadPromise.then(() => cb(null), cb) + } + + function _getFieldRoots(subtype) { + return fieldRoots.get(subtype) + } + + function _squeezePotential(subtype) { + const rootHash = FeedV1.getFeedRootHash(myWho, fromSubtype(subtype)) + const tangle = peer.db.getTangle(rootHash) + const maxDepth = tangle.getMaxDepth() + const fieldRoots = _getFieldRoots(subtype) + let minDepth = Infinity + for (const field in fieldRoots) { + for (const msgHash of fieldRoots[field]) { + const depth = tangle.getDepth(msgHash) + if (depth < minDepth) minDepth = depth + } + } + return maxDepth - minDepth + } + + function forceUpdate(subtype, update, cb) { + const type = fromSubtype(subtype) + + // Populate supersedes + const supersedes = [] + for (const field in update) { + const existing = fieldRoots.get(subtype, field) + if (existing) supersedes.push(...existing) + } + + peer.db.create({ type, content: { update, supersedes } }, (err, rec) => { + // prettier-ignore + if (err) return cb(new Error('Failed to create msg when force updating Record', { cause: err })) + cb(null, true) + }) + } + //#endregion + + //#region public methods + function get(authorId, subtype) { + const type = fromSubtype(subtype) + const rootHash = FeedV1.getFeedRootHash(authorId, type) + const tangle = peer.db.getTangle(rootHash) + if (!tangle || tangle.size() === 0) return {} + const msgHashes = tangle.topoSort() + const record = {} + for (const msgHash of msgHashes) { + const msg = peer.db.get(msgHash) + if (isValidRecordMsg(msg)) { + const { update } = msg.content + Object.assign(record, update) + } + } + return record + } + + function update(authorId, subtype, update, cb) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error('Cannot update another user\'s record. Given "authorId" was ' + authorId)) + + loaded(() => { + const record = get(authorId, subtype) + + let hasChanges = false + for (const [field, value] of Object.entries(update)) { + if (value !== record[field]) { + hasChanges = true + break + } + } + if (!hasChanges) return cb(null, false) + forceUpdate(subtype, update, cb) + }) + } + + function squeeze(authorId, subtype, cb) { + const who = FeedV1.stripAuthor(authorId) + // prettier-ignore + if (who !== myWho) return cb(new Error('Cannot squeeze another user\'s record. Given "authorId" was ' + authorId)) + const potential = _squeezePotential(subtype) + if (potential < 1) return cb(null, false) + + loaded(() => { + const record = get(authorId, subtype) + forceUpdate(subtype, record, (err) => { + // prettier-ignore + if (err) return cb(new Error('Failed to force update when squeezing Record', { cause: err })) + cb(null, true) + }) + }) + } + //#endregion + + return { + update, + get, + squeeze, + + _getFieldRoots, + _squeezePotential, + } + }, +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..41192a4 --- /dev/null +++ b/package.json @@ -0,0 +1,53 @@ +{ + "name": "ppppp-record", + "version": "1.0.0", + "description": "Record data structure over append-only logs with pruning", + "author": "Andre Staltz ", + "license": "MIT", + "homepage": "https://github.com/staltz/ppppp-record", + "repository": { + "type": "git", + "url": "git@github.com:staltz/ppppp-record.git" + }, + "main": "index.js", + "files": [ + "*.js", + "lib/*.js" + ], + "exports": { + ".": { + "require": "./lib/index.js" + } + }, + "type": "commonjs", + "engines": { + "node": ">=16" + }, + "dependencies": { + }, + "devDependencies": { + "bs58": "^5.0.0", + "c8": "7", + "ppppp-db": "github:staltz/ppppp-db", + "rimraf": "^4.4.0", + "secret-stack": "^6.4.1", + "ssb-box": "^1.0.1", + "ssb-caps": "^1.1.0", + "ssb-classic": "^1.1.0", + "ssb-keys": "^8.5.0", + "ssb-uri2": "^2.4.1", + "tap-arc": "^0.3.5", + "tape": "^5.6.3" + }, + "scripts": { + "test": "tape test/*.js | tap-arc --bail", + "format-code": "prettier --write \"(lib|test)/**/*.js\"", + "format-code-staged": "pretty-quick --staged --pattern \"(lib|test)/**/*.js\"", + "coverage": "c8 --reporter=lcov npm run test" + }, + "husky": { + "hooks": { + "pre-commit": "npm run format-code-staged" + } + } +} diff --git a/protospec.md b/protospec.md new file mode 100644 index 0000000..3ba8d07 --- /dev/null +++ b/protospec.md @@ -0,0 +1,132 @@ +## Feed tangle + +(Lipmaa backlinks are not shown in the diagram below, but they should exist) + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age] +B[updates name] +C[updates age] +D[updates name] +E[updates age & name] + +C-->B-->A-->R +D--->A +E-->D & C +``` + +Reducing the tangle above in a topological sort allows you to build a record +(a JSON object) `{age, name}`. + +## Msg type + +`msg.metadata.type` MUST start with `record_v1__`. E.g. `record_v1__profile`. + +## Msg content + +`msg.content` format: + +```typescript +interface MsgContent { + update: Record, + supersedes: Array, +} +``` + +RECOMMENDED that the `msg.content.update` is as flat as possible (no nesting). + +## Supersedes links + +When you update a field in a record, in the `supersedes` array you MUST point +to the currently-known highest-depth msg that updated that field. + +The set of *not-transitively-superseded-by-anyone* msgs comprise the +"field roots" of the record. To allow pruning the tangle, we can delete +(or, if we want to keep metadata, "erase") all msgs preceding the field roots. + +Suppose the tangle is grown in the order below, then the field roots are +highlighted in blue. + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age]:::blue +A-->R +classDef blue fill:#69f,stroke:#fff0,color:#000 +``` + +---- + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age]:::blue +B[updates name]:::blue +B-->A-->R +classDef blue fill:#69f,stroke:#fff0,color:#000 +``` + +----- + + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age] +B[updates name]:::blue +C[updates age]:::blue + +C-->B-->A-->R +C-- supersedes -->A + +linkStyle 3 stroke-width:1px,stroke:#05f +classDef blue fill:#69f,stroke:#fff0,color:#000 +``` + +----- + + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age] +B[updates name]:::blue +C[updates age]:::blue +D[updates name]:::blue + +C-->B-->A-->R +D--->A +C-- supersedes -->A + +linkStyle 4 stroke-width:1px,stroke:#05f +classDef blue fill:#69f,stroke:#fff0,color:#000 +``` +----- + + +```mermaid +graph RL + +R["(Feed root)"]:::feedroot +A[updates age] +B[updates name] +C[updates age] +D[updates name] +E[updates age & name]:::blue + +C-->B-->A-->R +C-- supersedes -->A +D-->A +E-->D & C +E-- supersedes -->C +E-- supersedes -->D + +linkStyle 3,7,8 stroke-width:1px,stroke:#05f +classDef blue fill:#69f,stroke:#fff0,color:#000 +``` diff --git a/test/index.test.js b/test/index.test.js new file mode 100644 index 0000000..ca0de25 --- /dev/null +++ b/test/index.test.js @@ -0,0 +1,145 @@ +const test = require('tape') +const path = require('path') +const os = require('os') +const rimraf = require('rimraf') +const SecretStack = require('secret-stack') +const FeedV1 = require('ppppp-db/feed-v1') +const caps = require('ssb-caps') +const p = require('util').promisify +const { generateKeypair } = require('./util') + +const DIR = path.join(os.tmpdir(), 'ppppp-record') +rimraf.sync(DIR) + +const aliceKeys = generateKeypair('alice') +const who = aliceKeys.id + +let peer +test('setup', async (t) => { + peer = SecretStack({ appKey: caps.shs }) + .use(require('ppppp-db')) + .use(require('ssb-box')) + .use(require('../lib')) + .call(null, { + keys: aliceKeys, + path: DIR, + }) + + await peer.db.loaded() +}) + +test('Record update() and get()', async (t) => { + t.ok( + await p(peer.record.update)(who, 'profile', { name: 'alice' }), + 'update .name' + ) + t.deepEqual(peer.record.get(who, 'profile'), { name: 'alice' }, 'get') + + const fieldRoots1 = peer.record._getFieldRoots('profile') + t.deepEquals(fieldRoots1, { name: ['Pt4YwxksvCLir45Tmw3hXK'] }, 'fieldRoots') + + t.ok(await p(peer.record.update)(who, 'profile', { age: 20 }), 'update .age') + t.deepEqual( + peer.record.get(who, 'profile'), + { name: 'alice', age: 20 }, + 'get' + ) + + const fieldRoots2 = peer.record._getFieldRoots('profile') + t.deepEquals( + fieldRoots2, + { name: ['Pt4YwxksvCLir45Tmw3hXK'], age: ['XqkG9Uz1eQcxv9R1f3jgKS'] }, + 'fieldRoots' + ) + + t.false( + await p(peer.record.update)(who, 'profile', { name: 'alice' }), + 'redundant update .name' + ) + t.deepEqual( + peer.record.get(who, 'profile'), + { name: 'alice', age: 20 }, + 'get' + ) + + t.true( + await p(peer.record.update)(who, 'profile', { name: 'Alice' }), + 'update .name' + ) + t.deepEqual( + peer.record.get(who, 'profile'), + { name: 'Alice', age: 20 }, + 'get' + ) + + const fieldRoots3 = peer.record._getFieldRoots('profile') + t.deepEquals( + fieldRoots3, + { name: ['WGDGt1UEGPpRyutfDyC2we'], age: ['XqkG9Uz1eQcxv9R1f3jgKS'] }, + 'fieldRoots' + ) +}) + +test('Record squeeze', async (t) => { + t.ok(await p(peer.record.update)(who, 'profile', { age: 21 }), 'update .age') + t.ok(await p(peer.record.update)(who, 'profile', { age: 22 }), 'update .age') + t.ok(await p(peer.record.update)(who, 'profile', { age: 23 }), 'update .age') + + const fieldRoots4 = peer.record._getFieldRoots('profile') + t.deepEquals( + fieldRoots4, + { name: ['WGDGt1UEGPpRyutfDyC2we'], age: ['6qu5mbLbFPJHCFge7QtU48'] }, + 'fieldRoots' + ) + + t.equals(peer.record._squeezePotential('profile'), 3, 'squeezePotential=3') + t.true(await p(peer.record.squeeze)(who, 'profile'), 'squeezed') + + const fieldRoots5 = peer.record._getFieldRoots('profile') + t.deepEquals( + fieldRoots5, + { name: ['Ba96TjutuuPbdMMvNS4BbL'], age: ['Ba96TjutuuPbdMMvNS4BbL'] }, + 'fieldRoots' + ) + + t.equals(peer.record._squeezePotential('profile'), 0, 'squeezePotential=0') + t.false(await p(peer.record.squeeze)(who, 'profile'), 'squeeze idempotent') + + const fieldRoots6 = peer.record._getFieldRoots('profile') + t.deepEquals(fieldRoots6, fieldRoots5, 'fieldRoots') +}) + +test('Record receives old branched update', async (t) => { + const rootMsg = FeedV1.createRoot(aliceKeys, 'record_v1__profile') + const rootHash = FeedV1.getMsgHash(rootMsg) + + const tangle = new FeedV1.Tangle(rootHash) + tangle.add(rootHash, rootMsg) + + const msg = FeedV1.create({ + keys: aliceKeys, + type: 'record_v1__profile', + content: { update: { age: 2 }, supersedes: [] }, + tangles: { + [rootHash]: tangle, + }, + }) + const rec = await p(peer.db.add)(msg, rootHash) + t.equals(rec.hash, 'JXvFSXE9s1DF77cSu5XUm', 'msg hash') + + const fieldRoots7 = peer.record._getFieldRoots('profile') + t.deepEquals( + fieldRoots7, + { + name: ['Ba96TjutuuPbdMMvNS4BbL'], + age: ['Ba96TjutuuPbdMMvNS4BbL', rec.hash], + }, + 'fieldRoots' + ) + + t.equals(peer.record._squeezePotential('profile'), 6, 'squeezePotential=6') +}) + +test('teardown', (t) => { + peer.close(true, t.end) +}) diff --git a/test/util.js b/test/util.js new file mode 100644 index 0000000..51feb19 --- /dev/null +++ b/test/util.js @@ -0,0 +1,14 @@ +const ssbKeys = require('ssb-keys') +const SSBURI = require('ssb-uri2') +const base58 = require('bs58') + +function generateKeypair(seed) { + const keys = ssbKeys.generate('ed25519', seed, 'buttwoo-v1') + const { data } = SSBURI.decompose(keys.id) + keys.id = `ppppp:feed/v1/${base58.encode(Buffer.from(data, 'base64'))}` + return keys +} + +module.exports = { + generateKeypair, +}