From d74695b7e7c97ee7642eef397b4f446abc64f192 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 18 Oct 2023 17:04:50 +0300 Subject: [PATCH] add ghosts API --- lib/index.js | 65 +++++++++++++++++++++++- lib/utils.js | 114 ++++++++++++++++++++++++++++++++++++++++++- package.json | 1 + test/ghosts.tests.js | 56 +++++++++++++++++++++ 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 test/ghosts.tests.js diff --git a/lib/index.js b/lib/index.js index 14d9cd9..424e42d 100644 --- a/lib/index.js +++ b/lib/index.js @@ -15,7 +15,7 @@ const { ACCOUNT_SELF, ACCOUNT_ANY, } = require('./msg-v3/constants') -const { ReadyGate } = require('./utils') +const { ReadyGate, GhostDB } = require('./utils') const { decrypt } = require('./encryption') /** @@ -162,6 +162,8 @@ function initDB(peer, config) { }, }) + const ghostDB = new GhostDB(Path.join(config.path, 'ghosts')) + peer.close.hook(function (/** @type {any} */ fn, /** @type {any} */ args) { log.close(() => { // @ts-ignore @@ -929,6 +931,62 @@ function initDB(peer, config) { }) } + /** + * @param {{ msg: MsgID; tangle: MsgID; max: number; }} opts + * @param {CB} cb + */ + function addGhost(opts, cb) { + if (!opts) return cb(new Error('ghosts.add() requires an `opts`')) + // prettier-ignore + if (!opts.msg || typeof opts.msg !== 'string') return cb(new Error('ghosts.add() requires msgID of the deleted msg in `opts.msg`')) + // prettier-ignore + if (!opts.tangle || typeof opts.tangle !== 'string') return cb(new Error('ghosts.add() requires tangleID for the deleted msg in `opts.tangle`')) + // prettier-ignore + if (!opts.max || typeof opts.max !== 'number') return cb(new Error('ghosts.add() requires max depth distance in `opts.max`')) + const { msg: msgID, tangle: tangleID, max } = opts + const rec = getRecord(msgID) + if (!rec) return cb() + if (!rec.msg) return cb() + const tangleData = rec.msg.metadata.tangles[tangleID] + // prettier-ignore + if (!tangleData) return cb(new Error(`ghosts.add() opts.msg "${opts.msg}" does not belong to opts.tangle "${opts.tangle}"`)) + const depth = tangleData.depth + + ghostDB.save(tangleID, msgID, depth, max, (err) => { + // prettier-ignore + if (err) cb(new Error('ghosts.add() failed to save to disk', { cause: err })) + else cb() + }) + } + + /** + * @param {MsgID} tangleID + * @param {CB>} cb + */ + function getGhosts(tangleID, cb) { + ghostDB.read(tangleID, (err, ghosts) => { + if (err) return cb(new Error('ghosts.get() failed', { cause: err })) + const msgIDs = [...ghosts.keys()] + cb(null, msgIDs) + }) + } + + /** + * @param {MsgID} tangleID + * @param {CB} cb + */ + function getMinGhostDepth(tangleID, cb) { + ghostDB.read(tangleID, (err, ghosts) => { + // prettier-ignore + if (err) return cb(new Error('ghosts.getMinDepth() failed', { cause: err })) + let minDepth = Infinity + for (const depth of ghosts.values()) { + if (depth < minDepth) minDepth = depth + } + cb(null, minDepth) + }) + } + /** * @param {MsgID} msgID * @param {CB} cb @@ -993,6 +1051,11 @@ function initDB(peer, config) { get, del, erase, + ghosts: { + add: addGhost, + get: getGhosts, + getMinDepth: getMinGhostDepth, + }, onRecordAdded, getTangle, msgs, diff --git a/lib/utils.js b/lib/utils.js index 9ecb0ec..2419b67 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,3 +1,15 @@ +// @ts-ignore +const atomic = require('atomic-file-rw') +const Path = require('path') + +/** + * @template T + * @typedef {T extends void ? + * (...args: [Error] | []) => void : + * (...args: [Error] | [null, T]) => void + * } CB + */ + class ReadyGate { #waiting #ready @@ -21,4 +33,104 @@ class ReadyGate { } } -module.exports = { ReadyGate } +class GhostDB { + /** @type {string} */ + #basePath + + static encodingOpts = { encoding: 'utf-8' } + + /** + * @param {string} basePath + */ + constructor(basePath) { + this.#basePath = basePath + } + + /** + * @param {string} tangleID + */ + #path(tangleID) { + return Path.join(this.#basePath, tangleID) + } + + /** + * @param {Map} map + * @returns {string} + */ + #serialize(map) { + return JSON.stringify([...map]) + } + + /** + * @param {string} str + * @returns {Map} + */ + #deserialize(str) { + return new Map(JSON.parse(str)) + } + + /** + * @param {string} tangleID + * @param {string} msgID + * @param {number} depth + * @param {number} max + * @param {CB} cb + */ + save(tangleID, msgID, depth, max, cb) { + atomic.readFile( + this.#path(tangleID), + GhostDB.encodingOpts, + (/** @type {any} */ err, /** @type {any} */ str) => { + // Load Map + /** @type {Map} */ + let map; + if (err && err.code === 'ENOENT') map = new Map() + // prettier-ignore + else if (err) return cb(new Error('GhostDB.save() failed to read ghost file', { cause: err })) + else map = this.#deserialize(str) + + map.set(msgID, depth) + + // Garbage collect any ghost smaller than largestDepth - max + let largestDepth = -1 + for (const depth of map.values()) { + if (depth > largestDepth) largestDepth = depth + } + for (const [x, depth] of map.entries()) { + if (depth <= largestDepth - max) map.delete(x) + } + + atomic.writeFile( + this.#path(tangleID), + this.#serialize(map), + GhostDB.encodingOpts, + (/** @type {any} */ err) => { + // prettier-ignore + if (err) return cb(new Error('GhostDB.save() failed to write ghost file', { cause: err })) + else cb() + } + ) + } + ) + } + + /** + * @param {string} tangleID + * @param {CB>} cb + */ + read(tangleID, cb) { + atomic.readFile( + this.#path(tangleID), + GhostDB.encodingOpts, + (/** @type {any} */ err, /** @type {any} */ str) => { + // prettier-ignore + if (err) return cb(new Error('GhostDB.read() failed to read ghost file', { cause: err })) + const map = this.#deserialize(str) + + cb(null, map) + } + ) + } +} + +module.exports = { ReadyGate, GhostDB } diff --git a/package.json b/package.json index 2b001f4..b917730 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "async-append-only-log": "~4.3.10", + "atomic-file-rw": "~0.3.0", "blake3": "~2.1.7", "b4a": "~1.6.4", "bs58": "~5.0.0", diff --git a/test/ghosts.tests.js b/test/ghosts.tests.js new file mode 100644 index 0000000..506acd9 --- /dev/null +++ b/test/ghosts.tests.js @@ -0,0 +1,56 @@ +const test = require('node:test') +const assert = require('node:assert') +const path = require('node:path') +const os = require('node:os') +const p = require('node:util').promisify +const rimraf = require('rimraf') +const SecretStack = require('secret-stack') +const caps = require('ppppp-caps') +const Keypair = require('ppppp-keypair') + +const DIR = path.join(os.tmpdir(), 'ppppp-db-ghosts') +rimraf.sync(DIR) + +const keypair = Keypair.generate('ed25519', 'alice') +test('ghosts.add, ghosts.get, ghosts.getMinDepth', async (t) => { + const peer = SecretStack({ appKey: caps.shse }) + .use(require('../lib')) + .use(require('ssb-box')) + .call(null, { keypair, path: DIR }) + + await peer.db.loaded() + + const account = await p(peer.db.account.create)({ domain: 'person' }) + const MAX = 5 + + let msgIDs = [] + for (let i = 0; i < 10; i++) { + const rec = await p(peer.db.feed.publish)({ + account, + domain: 'post', + data: { text: 'hello ' + i }, + }) + msgIDs.push(rec.id) + } + const feedID = peer.db.feed.getID(account, 'post') + + await p(peer.db.ghosts.add)({ msg: msgIDs[0], tangle: feedID, max: MAX }) + await p(peer.db.ghosts.add)({ msg: msgIDs[1], tangle: feedID, max: MAX }) + await p(peer.db.ghosts.add)({ msg: msgIDs[2], tangle: feedID, max: MAX }) + await p(peer.db.ghosts.add)({ msg: msgIDs[3], tangle: feedID, max: MAX }) + await p(peer.db.ghosts.add)({ msg: msgIDs[4], tangle: feedID, max: MAX }) + + const ghostsA = await p(peer.db.ghosts.get)(feedID) + assert.deepEqual(ghostsA, msgIDs.slice(0, 5), 'ghosts so far') + const depthA = await p(peer.db.ghosts.getMinDepth)(feedID) + assert.equal(depthA, 1, 'min depth so far') + + await p(peer.db.ghosts.add)({ msg: msgIDs[5], tangle: feedID, max: MAX }) + + const ghostsB = await p(peer.db.ghosts.get)(feedID) + assert.deepEqual(ghostsB, msgIDs.slice(1, 6), 'ghosts so far') + const depthB = await p(peer.db.ghosts.getMinDepth)(feedID) + assert.equal(depthB, 2, 'min depth so far') + + await p(peer.close)(true) +})