From 317f7c0a055b666238e958de9ce59b82d80c4a70 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Sep 2023 17:31:11 +0300 Subject: [PATCH] update to new ppppp-db with rec.id --- lib/algorithm.js | 96 +++++++++++++++++++-------------------- test/account-sync.test.js | 4 +- test/feed-sync.test.js | 46 +++++++++---------- test/thread-sync.test.js | 44 +++++++++--------- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/lib/algorithm.js b/lib/algorithm.js index 60de317..ad3f3a7 100644 --- a/lib/algorithm.js +++ b/lib/algorithm.js @@ -25,18 +25,18 @@ class Algorithm { this.#peer = peer } - haveRange(rootMsgHash) { - const rootMsg = this.#peer.db.get(rootMsgHash) + haveRange(rootID) { + const rootMsg = this.#peer.db.get(rootID) if (!rootMsg) return [1, 0] let minDepth = Number.MAX_SAFE_INTEGER let maxDepth = 0 for (const rec of this.#peer.db.records()) { - if (!rec.msg?.data) continue + if (!rec?.msg?.data) continue const tangles = rec.msg.metadata.tangles - if (rec.hash === rootMsgHash) { + if (rec.id === rootID) { minDepth = 0 - } else if (tangles[rootMsgHash]) { - const depth = tangles[rootMsgHash].depth + } else if (tangles[rootID]) { + const depth = tangles[rootID].depth minDepth = Math.min(minDepth, depth) maxDepth = Math.max(maxDepth, depth) } @@ -99,13 +99,13 @@ class Algorithm { } } - bloomFor(rootMsgHash, round, range, extraIds = []) { + bloomFor(rootID, round, range, extraIds = []) { const filterSize = (isEmptyRange(range) ? 2 : estimateMsgCount(range)) + countIter(extraIds) const filter = BloomFilter.create(2 * filterSize, 0.00001) if (!isEmptyRange(range)) { - for (const msg of this.yieldMsgsIn(rootMsgHash, range)) { - filter.add('' + round + MsgV3.getMsgHash(msg)) + for (const msg of this.yieldMsgsIn(rootID, range)) { + filter.add('' + round + MsgV3.getMsgID(msg)) } } for (const msgId of extraIds) { @@ -114,59 +114,59 @@ class Algorithm { return filter.saveAsJSON() } - msgsMissing(rootMsgHash, round, range, remoteBloomJSON) { + msgsMissing(rootID, round, range, remoteBloomJSON) { if (isEmptyRange(range)) return [] const remoteFilter = BloomFilter.fromJSON(remoteBloomJSON) const missing = [] - for (const msg of this.yieldMsgsIn(rootMsgHash, range)) { - const msgHash = MsgV3.getMsgHash(msg) - if (!remoteFilter.has('' + round + msgHash)) { - missing.push(msgHash) + for (const msg of this.yieldMsgsIn(rootID, range)) { + const msgID = MsgV3.getMsgID(msg) + if (!remoteFilter.has('' + round + msgID)) { + missing.push(msgID) } } return missing } - *yieldMsgsIn(rootMsgHash, range) { + *yieldMsgsIn(rootID, range) { const [minDepth, maxDepth] = range - const rootMsg = this.#peer.db.get(rootMsgHash) + const rootMsg = this.#peer.db.get(rootID) if (!rootMsg) return if (minDepth === 0) yield rootMsg for (const msg of this.#peer.db.msgs()) { const tangles = msg.metadata.tangles if ( - tangles[rootMsgHash] && - tangles[rootMsgHash].depth >= minDepth && - tangles[rootMsgHash].depth <= maxDepth + tangles[rootID] && + tangles[rootID].depth >= minDepth && + tangles[rootID].depth <= maxDepth ) { yield msg } } } - async pruneNewest(rootMsgHash, count) { - const tangle = this.#peer.db.getTangle(rootMsgHash) + async pruneNewest(rootID, count) { + const tangle = this.#peer.db.getTangle(rootID) const sorted = tangle.topoSort() if (sorted.length <= count) return - const msgHash = sorted[sorted.length - count] - const { deletables, erasables } = tangle.getDeletablesAndErasables(msgHash) + const msgID = sorted[sorted.length - count] + const { deletables, erasables } = tangle.getDeletablesAndErasables(msgID) const del = p(this.#peer.db.del) const erase = p(this.#peer.db.erase) - for (const msgHash of deletables) { - await del(msgHash) + for (const msgID of deletables) { + await del(msgID) } - for (const msgHash of erasables) { - await erase(msgHash) + for (const msgID of erasables) { + await erase(msgID) } } - async commit(rootMsgHash, newMsgs, goal, myWantRange) { + async commit(rootID, newMsgs, goal, myWantRange) { // Filter out dataful newMsgs that are not in my want-range const [minWant, maxWant] = myWantRange const validNewMsgs = newMsgs .filter((msg) => { - const depth = msg.metadata.tangles[rootMsgHash]?.depth ?? 0 - if (depth === 0 && MsgV3.getMsgHash(msg) !== rootMsgHash) { + const depth = msg.metadata.tangles[rootID]?.depth ?? 0 + if (depth === 0 && MsgV3.getMsgID(msg) !== rootID) { return false // the rootMsg is the only acceptable depth-zero msg } if (!msg.data) { @@ -176,8 +176,8 @@ class Algorithm { } }) .sort((a, b) => { - const aDepth = a.metadata.tangles[rootMsgHash]?.depth ?? 0 - const bDepth = b.metadata.tangles[rootMsgHash]?.depth ?? 0 + const aDepth = a.metadata.tangles[rootID]?.depth ?? 0 + const bDepth = b.metadata.tangles[rootID]?.depth ?? 0 return aDepth - bDepth }) @@ -187,39 +187,39 @@ class Algorithm { // TODO: optimize perf, avoiding await / try / catch for (const msg of validNewMsgs) { try { - await p(this.#peer.db.add)(msg, rootMsgHash) + await p(this.#peer.db.add)(msg, rootID) } catch {} } // Prune. Ideally this should be in a garbage collection module const { type, count } = parseGoal(goal) - if (type === 'newest') return await this.pruneNewest(rootMsgHash, count) + if (type === 'newest') return await this.pruneNewest(rootID, count) if (type === 'oldest') throw new Error('not implemented') // TODO: } /** - * @param {string} rootMsgHash - * @param {Set} msgHashes + * @param {string} rootID + * @param {Set} msgIDs * @returns */ - getTangleSlice(rootMsgHash, msgHashes) { - if (msgHashes.size === 0) return [] - const tangle = this.#peer.db.getTangle(rootMsgHash) + getTangleSlice(rootID, msgIDs) { + if (msgIDs.size === 0) return [] + const tangle = this.#peer.db.getTangle(rootID) const sorted = tangle.topoSort() - let oldestMsgHash = null - for (const msgHash of sorted) { - if (msgHashes.has(msgHash)) { - oldestMsgHash = msgHash + let oldestMsgID = null + for (const msgID of sorted) { + if (msgIDs.has(msgID)) { + oldestMsgID = msgID break } } - const { erasables } = tangle.getDeletablesAndErasables(oldestMsgHash) + const { erasables } = tangle.getDeletablesAndErasables(oldestMsgID) const msgs = [] - for (const msgHash of sorted) { - let isErasable = erasables.includes(msgHash) - if (!msgHashes.has(msgHash) && !isErasable) continue - const msg = this.#peer.db.get(msgHash) + for (const msgID of sorted) { + let isErasable = erasables.includes(msgID) + if (!msgIDs.has(msgID) && !isErasable) continue + const msg = this.#peer.db.get(msgID) if (!msg) continue if (isErasable) { msgs.push({ ...msg, data: null }) diff --git a/test/account-sync.test.js b/test/account-sync.test.js index 558130d..2292ecb 100644 --- a/test/account-sync.test.js +++ b/test/account-sync.test.js @@ -9,8 +9,8 @@ const bobKeys = Keypair.generate('ed25519', 'bob') function getAccount(iter) { return [...iter] - .filter((msg) => msg.metadata.account === 'self' && msg.data) - .map((msg) => msg.data.add.key.bytes) + .filter((m) => m.metadata.account === 'self' && m.data?.action === 'add') + .map((m) => m.data.add.key.bytes) } test('sync an account tangle', async (t) => { diff --git a/test/feed-sync.test.js b/test/feed-sync.test.js index 4a537c4..764263f 100644 --- a/test/feed-sync.test.js +++ b/test/feed-sync.test.js @@ -19,10 +19,10 @@ test('sync a feed with goal=all', async (t) => { domain: 'account', _nonce: 'carol', }) - const carolIDMsg = alice.db.get(carolID) + const carolAccountRoot = alice.db.get(carolID) // Bob knows Carol - await p(bob.db.add)(carolIDMsg, carolID) + await p(bob.db.add)(carolAccountRoot, carolID) const carolMsgs = [] for (let i = 1; i <= 10; i++) { @@ -36,12 +36,12 @@ test('sync a feed with goal=all', async (t) => { } assert('alice has msgs 1..10 from carol') - const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') - const carolPostsRootMsg = alice.db.get(carolPostsRootHash) + const carolPostsMootID = alice.db.feed.getID(carolID, 'post') + const carolPostsMoot = alice.db.get(carolPostsMootID) - await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) + await p(bob.db.add)(carolPostsMoot, carolPostsMootID) for (let i = 0; i < 7; i++) { - await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) + await p(bob.db.add)(carolMsgs[i], carolPostsMootID) } { @@ -55,8 +55,8 @@ test('sync a feed with goal=all', async (t) => { ) } - bob.tangleSync.setGoal(carolPostsRootHash, 'all') - alice.tangleSync.setGoal(carolPostsRootHash, 'all') + bob.tangleSync.setGoal(carolPostsMootID, 'all') + alice.tangleSync.setGoal(carolPostsMootID, 'all') const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('bob connected to alice') @@ -93,10 +93,10 @@ test('sync a feed with goal=newest', async (t) => { domain: 'account', _nonce: 'carol', }) - const carolIDMsg = alice.db.get(carolID) + const carolAccountRoot = alice.db.get(carolID) // Bob knows Carol - await p(bob.db.add)(carolIDMsg, carolID) + await p(bob.db.add)(carolAccountRoot, carolID) const carolMsgs = [] for (let i = 1; i <= 10; i++) { @@ -110,12 +110,12 @@ test('sync a feed with goal=newest', async (t) => { } assert('alice has msgs 1..10 from carol') - const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') - const carolPostsRootMsg = alice.db.get(carolPostsRootHash) + const carolPostsMootID = alice.db.feed.getID(carolID, 'post') + const carolPostsMoot = alice.db.get(carolPostsMootID) - await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) + await p(bob.db.add)(carolPostsMoot, carolPostsMootID) for (let i = 0; i < 7; i++) { - await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) + await p(bob.db.add)(carolMsgs[i], carolPostsMootID) } { @@ -129,8 +129,8 @@ test('sync a feed with goal=newest', async (t) => { ) } - bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5') - alice.tangleSync.setGoal(carolPostsRootHash, 'all') + bob.tangleSync.setGoal(carolPostsMootID, 'newest-5') + alice.tangleSync.setGoal(carolPostsMootID, 'all') const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('bob connected to alice') @@ -183,11 +183,11 @@ test('sync a feed with goal=newest but too far behind', async (t) => { carolMsgs.push(rec.msg) } - const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') - const carolPostsRootMsg = alice.db.get(carolPostsRootHash) + const carolPostsMootID = alice.db.feed.getID(carolID, 'post') + const carolPostsMoot = alice.db.get(carolPostsMootID) const algo = new Algorithm(alice) - await algo.pruneNewest(carolPostsRootHash, 5) + await algo.pruneNewest(carolPostsMootID, 5) { const arr = [...alice.db.msgs()] .filter((msg) => msg.metadata.account === carolID && msg.data) @@ -199,9 +199,9 @@ test('sync a feed with goal=newest but too far behind', async (t) => { ) } - await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) + await p(bob.db.add)(carolPostsMoot, carolPostsMootID) for (let i = 0; i < 2; i++) { - await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) + await p(bob.db.add)(carolMsgs[i], carolPostsMootID) } { @@ -211,8 +211,8 @@ test('sync a feed with goal=newest but too far behind', async (t) => { assert.deepEqual(arr, ['m1', 'm2'], 'bob has msgs 1..2 from carol') } - alice.tangleSync.setGoal(carolPostsRootHash, 'newest-5') - bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5') + alice.tangleSync.setGoal(carolPostsMootID, 'newest-5') + bob.tangleSync.setGoal(carolPostsMootID, 'newest-5') const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('bob connected to alice') diff --git a/test/thread-sync.test.js b/test/thread-sync.test.js index 6c617d0..9d98e22 100644 --- a/test/thread-sync.test.js +++ b/test/thread-sync.test.js @@ -95,7 +95,7 @@ test('sync a thread where both peers have portions', async (t) => { domain: 'post', data: { text: 'A' }, }) - const rootHashA = alice.db.feed.getId(aliceID, 'post') + const rootHashA = alice.db.feed.getID(aliceID, 'post') const rootMsgA = alice.db.get(rootHashA) await p(bob.db.add)(rootMsgA, rootHashA) @@ -105,16 +105,16 @@ test('sync a thread where both peers have portions', async (t) => { account: bobID, domain: 'post', data: { text: 'B1' }, - tangles: [startA.hash], + tangles: [startA.id], }) const replyB2 = await p(bob.db.feed.publish)({ account: bobID, domain: 'post', data: { text: 'B2' }, - tangles: [startA.hash], + tangles: [startA.id], }) - const rootHashB = bob.db.feed.getId(bobID, 'post') + const rootHashB = bob.db.feed.getID(bobID, 'post') const rootMsgB = bob.db.get(rootHashB) await p(alice.db.add)(rootMsgB, rootHashB) @@ -125,7 +125,7 @@ test('sync a thread where both peers have portions', async (t) => { account: carolID, domain: 'post', data: { text: 'C1' }, - tangles: [startA.hash], + tangles: [startA.id], keypair: carolKeypair, }) @@ -133,7 +133,7 @@ test('sync a thread where both peers have portions', async (t) => { account: daveID, domain: 'post', data: { text: 'D1' }, - tangles: [startA.hash], + tangles: [startA.id], keypair: daveKeypair, }) @@ -149,8 +149,8 @@ test('sync a thread where both peers have portions', async (t) => { 'bob has another portion of the thread' ) - bob.tangleSync.setGoal(startA.hash, 'all') - alice.tangleSync.setGoal(startA.hash, 'all') + bob.tangleSync.setGoal(startA.id, 'all') + alice.tangleSync.setGoal(startA.id, 'all') const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('bob connected to alice') @@ -210,14 +210,14 @@ test('sync a thread where initiator does not have the root', async (t) => { account: aliceID, domain: 'post', data: { text: 'A1' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) const replyA2 = await p(alice.db.feed.publish)({ account: aliceID, domain: 'post', data: { text: 'A2' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) assert.deepEqual( @@ -228,9 +228,9 @@ test('sync a thread where initiator does not have the root', async (t) => { assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') - bob.tangleSync.setGoal(rootA.hash, 'all') + bob.tangleSync.setGoal(rootA.id, 'all') // ON PURPOSE: alice does not set the goal - // alice.tangleSync.setGoal(rootA.hash, 'all') + // alice.tangleSync.setGoal(rootA.id, 'all') const remoteAlice = await p(bob.connect)(alice.getAddress()) assert('bob connected to alice') @@ -284,14 +284,14 @@ test('sync a thread where receiver does not have the root', async (t) => { account: aliceID, domain: 'post', data: { text: 'A1' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) const replyA2 = await p(alice.db.feed.publish)({ account: aliceID, domain: 'post', data: { text: 'A2' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) assert.deepEqual( @@ -302,8 +302,8 @@ test('sync a thread where receiver does not have the root', async (t) => { assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') - bob.tangleSync.setGoal(rootA.hash, 'all') - alice.tangleSync.setGoal(rootA.hash, 'all') + bob.tangleSync.setGoal(rootA.id, 'all') + alice.tangleSync.setGoal(rootA.id, 'all') const remoteBob = await p(alice.connect)(bob.getAddress()) assert('alice connected to bob') @@ -357,21 +357,21 @@ test('sync a thread with reactions too', async (t) => { account: aliceID, domain: 'post', data: { text: 'A1' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) const replyA2 = await p(alice.db.feed.publish)({ account: aliceID, domain: 'post', data: { text: 'A2' }, - tangles: [rootA.hash], + tangles: [rootA.id], }) const reactionA3 = await p(alice.db.feed.publish)({ account: aliceID, domain: 'reaction', - data: { text: 'yes', link: replyA1.hash }, - tangles: [rootA.hash, replyA1.hash], + data: { text: 'yes', link: replyA1.id }, + tangles: [rootA.id, replyA1.id], }) assert.deepEqual( @@ -382,8 +382,8 @@ test('sync a thread with reactions too', async (t) => { assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') - bob.tangleSync.setGoal(rootA.hash, 'all') - alice.tangleSync.setGoal(rootA.hash, 'all') + bob.tangleSync.setGoal(rootA.id, 'all') + alice.tangleSync.setGoal(rootA.id, 'all') const remoteBob = await p(alice.connect)(bob.getAddress()) assert('alice connected to bob')