pzp
/
pzp-sync
mirror of https://codeberg.org/pzp/pzp-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update to new ppppp-db with rec.id

This commit is contained in:
Andre Staltz 2023-09-07 17:31:11 +03:00
parent f6f16bdd47
commit 317f7c0a05
No known key found for this signature in database
GPG Key ID: 9EDE23EA7E8A4890
4 changed files with 95 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
lib/algorithm.js
View File

@ -25,18 +25,18 @@ class Algorithm {
this.#peer = peer this.#peer = peer
} }
haveRange(rootMsgHash) { haveRange(rootID) {
const rootMsg = this.#peer.db.get(rootMsgHash) const rootMsg = this.#peer.db.get(rootID)
if (!rootMsg) return [1, 0] if (!rootMsg) return [1, 0]
let minDepth = Number.MAX_SAFE_INTEGER let minDepth = Number.MAX_SAFE_INTEGER
let maxDepth = 0 let maxDepth = 0
for (const rec of this.#peer.db.records()) { for (const rec of this.#peer.db.records()) {
if (!rec.msg?.data) continue if (!rec?.msg?.data) continue
const tangles = rec.msg.metadata.tangles const tangles = rec.msg.metadata.tangles
if (rec.hash === rootMsgHash) { if (rec.id === rootID) {
minDepth = 0 minDepth = 0
} else if (tangles[rootMsgHash]) { } else if (tangles[rootID]) {
const depth = tangles[rootMsgHash].depth const depth = tangles[rootID].depth
minDepth = Math.min(minDepth, depth) minDepth = Math.min(minDepth, depth)
maxDepth = Math.max(maxDepth, depth) maxDepth = Math.max(maxDepth, depth)
} }
@ -99,13 +99,13 @@ class Algorithm {
} }
} }
bloomFor(rootMsgHash, round, range, extraIds = []) { bloomFor(rootID, round, range, extraIds = []) {
const filterSize = const filterSize =
(isEmptyRange(range) ? 2 : estimateMsgCount(range)) + countIter(extraIds) (isEmptyRange(range) ? 2 : estimateMsgCount(range)) + countIter(extraIds)
const filter = BloomFilter.create(2 * filterSize, 0.00001) const filter = BloomFilter.create(2 * filterSize, 0.00001)
if (!isEmptyRange(range)) { if (!isEmptyRange(range)) {
for (const msg of this.yieldMsgsIn(rootMsgHash, range)) { for (const msg of this.yieldMsgsIn(rootID, range)) {
filter.add('' + round + MsgV3.getMsgHash(msg)) filter.add('' + round + MsgV3.getMsgID(msg))
} }
} }
for (const msgId of extraIds) { for (const msgId of extraIds) {
@ -114,59 +114,59 @@ class Algorithm {
return filter.saveAsJSON() return filter.saveAsJSON()
} }
msgsMissing(rootMsgHash, round, range, remoteBloomJSON) { msgsMissing(rootID, round, range, remoteBloomJSON) {
if (isEmptyRange(range)) return [] if (isEmptyRange(range)) return []
const remoteFilter = BloomFilter.fromJSON(remoteBloomJSON) const remoteFilter = BloomFilter.fromJSON(remoteBloomJSON)
const missing = [] const missing = []
for (const msg of this.yieldMsgsIn(rootMsgHash, range)) { for (const msg of this.yieldMsgsIn(rootID, range)) {
const msgHash = MsgV3.getMsgHash(msg) const msgID = MsgV3.getMsgID(msg)
if (!remoteFilter.has('' + round + msgHash)) { if (!remoteFilter.has('' + round + msgID)) {
missing.push(msgHash) missing.push(msgID)
} }
} }
return missing return missing
} }
*yieldMsgsIn(rootMsgHash, range) { *yieldMsgsIn(rootID, range) {
const [minDepth, maxDepth] = range const [minDepth, maxDepth] = range
const rootMsg = this.#peer.db.get(rootMsgHash) const rootMsg = this.#peer.db.get(rootID)
if (!rootMsg) return if (!rootMsg) return
if (minDepth === 0) yield rootMsg if (minDepth === 0) yield rootMsg
for (const msg of this.#peer.db.msgs()) { for (const msg of this.#peer.db.msgs()) {
const tangles = msg.metadata.tangles const tangles = msg.metadata.tangles
if ( if (
tangles[rootMsgHash] && tangles[rootID] &&
tangles[rootMsgHash].depth >= minDepth && tangles[rootID].depth >= minDepth &&
tangles[rootMsgHash].depth <= maxDepth tangles[rootID].depth <= maxDepth
) { ) {
yield msg yield msg
} }
} }
} }
async pruneNewest(rootMsgHash, count) { async pruneNewest(rootID, count) {
const tangle = this.#peer.db.getTangle(rootMsgHash) const tangle = this.#peer.db.getTangle(rootID)
const sorted = tangle.topoSort() const sorted = tangle.topoSort()
if (sorted.length <= count) return if (sorted.length <= count) return
const msgHash = sorted[sorted.length - count] const msgID = sorted[sorted.length - count]
const { deletables, erasables } = tangle.getDeletablesAndErasables(msgHash) const { deletables, erasables } = tangle.getDeletablesAndErasables(msgID)
const del = p(this.#peer.db.del) const del = p(this.#peer.db.del)
const erase = p(this.#peer.db.erase) const erase = p(this.#peer.db.erase)
for (const msgHash of deletables) { for (const msgID of deletables) {
await del(msgHash) await del(msgID)
} }
for (const msgHash of erasables) { for (const msgID of erasables) {
await erase(msgHash) await erase(msgID)
} }
} }
async commit(rootMsgHash, newMsgs, goal, myWantRange) { async commit(rootID, newMsgs, goal, myWantRange) {
// Filter out dataful newMsgs that are not in my want-range // Filter out dataful newMsgs that are not in my want-range
const [minWant, maxWant] = myWantRange const [minWant, maxWant] = myWantRange
const validNewMsgs = newMsgs const validNewMsgs = newMsgs
.filter((msg) => { .filter((msg) => {
const depth = msg.metadata.tangles[rootMsgHash]?.depth ?? 0 const depth = msg.metadata.tangles[rootID]?.depth ?? 0
if (depth === 0 && MsgV3.getMsgHash(msg) !== rootMsgHash) { if (depth === 0 && MsgV3.getMsgID(msg) !== rootID) {
return false // the rootMsg is the only acceptable depth-zero msg return false // the rootMsg is the only acceptable depth-zero msg
} }
if (!msg.data) { if (!msg.data) {
@ -176,8 +176,8 @@ class Algorithm {
} }
}) })
.sort((a, b) => { .sort((a, b) => {
const aDepth = a.metadata.tangles[rootMsgHash]?.depth ?? 0 const aDepth = a.metadata.tangles[rootID]?.depth ?? 0
const bDepth = b.metadata.tangles[rootMsgHash]?.depth ?? 0 const bDepth = b.metadata.tangles[rootID]?.depth ?? 0
return aDepth - bDepth return aDepth - bDepth
}) })
@ -187,39 +187,39 @@ class Algorithm {
// TODO: optimize perf, avoiding await / try / catch // TODO: optimize perf, avoiding await / try / catch
for (const msg of validNewMsgs) { for (const msg of validNewMsgs) {
try { try {
await p(this.#peer.db.add)(msg, rootMsgHash) await p(this.#peer.db.add)(msg, rootID)
} catch {} } catch {}
} }
// Prune. Ideally this should be in a garbage collection module // Prune. Ideally this should be in a garbage collection module
const { type, count } = parseGoal(goal) const { type, count } = parseGoal(goal)
if (type === 'newest') return await this.pruneNewest(rootMsgHash, count) if (type === 'newest') return await this.pruneNewest(rootID, count)
if (type === 'oldest') throw new Error('not implemented') // TODO: if (type === 'oldest') throw new Error('not implemented') // TODO:
} }
/** /**
* @param {string} rootMsgHash * @param {string} rootID
* @param {Set<string>} msgHashes * @param {Set<string>} msgIDs
* @returns * @returns
*/ */
getTangleSlice(rootMsgHash, msgHashes) { getTangleSlice(rootID, msgIDs) {
if (msgHashes.size === 0) return [] if (msgIDs.size === 0) return []
const tangle = this.#peer.db.getTangle(rootMsgHash) const tangle = this.#peer.db.getTangle(rootID)
const sorted = tangle.topoSort() const sorted = tangle.topoSort()
let oldestMsgHash = null let oldestMsgID = null
for (const msgHash of sorted) { for (const msgID of sorted) {
if (msgHashes.has(msgHash)) { if (msgIDs.has(msgID)) {
oldestMsgHash = msgHash oldestMsgID = msgID
break break
} }
} }
const { erasables } = tangle.getDeletablesAndErasables(oldestMsgHash) const { erasables } = tangle.getDeletablesAndErasables(oldestMsgID)
const msgs = [] const msgs = []
for (const msgHash of sorted) { for (const msgID of sorted) {
let isErasable = erasables.includes(msgHash) let isErasable = erasables.includes(msgID)
if (!msgHashes.has(msgHash) && !isErasable) continue if (!msgIDs.has(msgID) && !isErasable) continue
const msg = this.#peer.db.get(msgHash) const msg = this.#peer.db.get(msgID)
if (!msg) continue if (!msg) continue
if (isErasable) { if (isErasable) {
msgs.push({ ...msg, data: null }) msgs.push({ ...msg, data: null })

4
test/account-sync.test.js
View File

@ -9,8 +9,8 @@ const bobKeys = Keypair.generate('ed25519', 'bob')
function getAccount(iter) { function getAccount(iter) {
return [...iter] return [...iter]
.filter((msg) => msg.metadata.account === 'self' && msg.data) .filter((m) => m.metadata.account === 'self' && m.data?.action === 'add')
.map((msg) => msg.data.add.key.bytes) .map((m) => m.data.add.key.bytes)
} }
test('sync an account tangle', async (t) => { test('sync an account tangle', async (t) => {

46
test/feed-sync.test.js
View File

@ -19,10 +19,10 @@ test('sync a feed with goal=all', async (t) => {
domain: 'account', domain: 'account',
_nonce: 'carol', _nonce: 'carol',
}) })
const carolIDMsg = alice.db.get(carolID) const carolAccountRoot = alice.db.get(carolID)
// Bob knows Carol // Bob knows Carol
await p(bob.db.add)(carolIDMsg, carolID) await p(bob.db.add)(carolAccountRoot, carolID)
const carolMsgs = [] const carolMsgs = []
for (let i = 1; i <= 10; i++) { for (let i = 1; i <= 10; i++) {
@ -36,12 +36,12 @@ test('sync a feed with goal=all', async (t) => {
} }
assert('alice has msgs 1..10 from carol') assert('alice has msgs 1..10 from carol')
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash) const carolPostsMoot = alice.db.get(carolPostsMootID)
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 7; i++) { for (let i = 0; i < 7; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
} }
{ {
@ -55,8 +55,8 @@ test('sync a feed with goal=all', async (t) => {
) )
} }
bob.tangleSync.setGoal(carolPostsRootHash, 'all') bob.tangleSync.setGoal(carolPostsMootID, 'all')
alice.tangleSync.setGoal(carolPostsRootHash, 'all') alice.tangleSync.setGoal(carolPostsMootID, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress()) const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice') assert('bob connected to alice')
@ -93,10 +93,10 @@ test('sync a feed with goal=newest', async (t) => {
domain: 'account', domain: 'account',
_nonce: 'carol', _nonce: 'carol',
}) })
const carolIDMsg = alice.db.get(carolID) const carolAccountRoot = alice.db.get(carolID)
// Bob knows Carol // Bob knows Carol
await p(bob.db.add)(carolIDMsg, carolID) await p(bob.db.add)(carolAccountRoot, carolID)
const carolMsgs = [] const carolMsgs = []
for (let i = 1; i <= 10; i++) { for (let i = 1; i <= 10; i++) {
@ -110,12 +110,12 @@ test('sync a feed with goal=newest', async (t) => {
} }
assert('alice has msgs 1..10 from carol') assert('alice has msgs 1..10 from carol')
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash) const carolPostsMoot = alice.db.get(carolPostsMootID)
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 7; i++) { for (let i = 0; i < 7; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
} }
{ {
@ -129,8 +129,8 @@ test('sync a feed with goal=newest', async (t) => {
) )
} }
bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5') bob.tangleSync.setGoal(carolPostsMootID, 'newest-5')
alice.tangleSync.setGoal(carolPostsRootHash, 'all') alice.tangleSync.setGoal(carolPostsMootID, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress()) const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice') assert('bob connected to alice')
@ -183,11 +183,11 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
carolMsgs.push(rec.msg) carolMsgs.push(rec.msg)
} }
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post') const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash) const carolPostsMoot = alice.db.get(carolPostsMootID)
const algo = new Algorithm(alice) const algo = new Algorithm(alice)
await algo.pruneNewest(carolPostsRootHash, 5) await algo.pruneNewest(carolPostsMootID, 5)
{ {
const arr = [...alice.db.msgs()] const arr = [...alice.db.msgs()]
.filter((msg) => msg.metadata.account === carolID && msg.data) .filter((msg) => msg.metadata.account === carolID && msg.data)
@ -199,9 +199,9 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
) )
} }
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash) await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 2; i++) { for (let i = 0; i < 2; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash) await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
} }
{ {
@ -211,8 +211,8 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
assert.deepEqual(arr, ['m1', 'm2'], 'bob has msgs 1..2 from carol') assert.deepEqual(arr, ['m1', 'm2'], 'bob has msgs 1..2 from carol')
} }
alice.tangleSync.setGoal(carolPostsRootHash, 'newest-5') alice.tangleSync.setGoal(carolPostsMootID, 'newest-5')
bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5') bob.tangleSync.setGoal(carolPostsMootID, 'newest-5')
const remoteAlice = await p(bob.connect)(alice.getAddress()) const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice') assert('bob connected to alice')

44
test/thread-sync.test.js
View File

@ -95,7 +95,7 @@ test('sync a thread where both peers have portions', async (t) => {
domain: 'post', domain: 'post',
data: { text: 'A' }, data: { text: 'A' },
}) })
const rootHashA = alice.db.feed.getId(aliceID, 'post') const rootHashA = alice.db.feed.getID(aliceID, 'post')
const rootMsgA = alice.db.get(rootHashA) const rootMsgA = alice.db.get(rootHashA)
await p(bob.db.add)(rootMsgA, rootHashA) await p(bob.db.add)(rootMsgA, rootHashA)
@ -105,16 +105,16 @@ test('sync a thread where both peers have portions', async (t) => {
account: bobID, account: bobID,
domain: 'post', domain: 'post',
data: { text: 'B1' }, data: { text: 'B1' },
tangles: [startA.hash], tangles: [startA.id],
}) })
const replyB2 = await p(bob.db.feed.publish)({ const replyB2 = await p(bob.db.feed.publish)({
account: bobID, account: bobID,
domain: 'post', domain: 'post',
data: { text: 'B2' }, data: { text: 'B2' },
tangles: [startA.hash], tangles: [startA.id],
}) })
const rootHashB = bob.db.feed.getId(bobID, 'post') const rootHashB = bob.db.feed.getID(bobID, 'post')
const rootMsgB = bob.db.get(rootHashB) const rootMsgB = bob.db.get(rootHashB)
await p(alice.db.add)(rootMsgB, rootHashB) await p(alice.db.add)(rootMsgB, rootHashB)
@ -125,7 +125,7 @@ test('sync a thread where both peers have portions', async (t) => {
account: carolID, account: carolID,
domain: 'post', domain: 'post',
data: { text: 'C1' }, data: { text: 'C1' },
tangles: [startA.hash], tangles: [startA.id],
keypair: carolKeypair, keypair: carolKeypair,
}) })
@ -133,7 +133,7 @@ test('sync a thread where both peers have portions', async (t) => {
account: daveID, account: daveID,
domain: 'post', domain: 'post',
data: { text: 'D1' }, data: { text: 'D1' },
tangles: [startA.hash], tangles: [startA.id],
keypair: daveKeypair, keypair: daveKeypair,
}) })
@ -149,8 +149,8 @@ test('sync a thread where both peers have portions', async (t) => {
'bob has another portion of the thread' 'bob has another portion of the thread'
) )
bob.tangleSync.setGoal(startA.hash, 'all') bob.tangleSync.setGoal(startA.id, 'all')
alice.tangleSync.setGoal(startA.hash, 'all') alice.tangleSync.setGoal(startA.id, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress()) const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice') assert('bob connected to alice')
@ -210,14 +210,14 @@ test('sync a thread where initiator does not have the root', async (t) => {
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A1' }, data: { text: 'A1' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
const replyA2 = await p(alice.db.feed.publish)({ const replyA2 = await p(alice.db.feed.publish)({
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A2' }, data: { text: 'A2' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
assert.deepEqual( assert.deepEqual(
@ -228,9 +228,9 @@ test('sync a thread where initiator does not have the root', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all') bob.tangleSync.setGoal(rootA.id, 'all')
// ON PURPOSE: alice does not set the goal // ON PURPOSE: alice does not set the goal
// alice.tangleSync.setGoal(rootA.hash, 'all') // alice.tangleSync.setGoal(rootA.id, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress()) const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice') assert('bob connected to alice')
@ -284,14 +284,14 @@ test('sync a thread where receiver does not have the root', async (t) => {
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A1' }, data: { text: 'A1' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
const replyA2 = await p(alice.db.feed.publish)({ const replyA2 = await p(alice.db.feed.publish)({
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A2' }, data: { text: 'A2' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
assert.deepEqual( assert.deepEqual(
@ -302,8 +302,8 @@ test('sync a thread where receiver does not have the root', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all') bob.tangleSync.setGoal(rootA.id, 'all')
alice.tangleSync.setGoal(rootA.hash, 'all') alice.tangleSync.setGoal(rootA.id, 'all')
const remoteBob = await p(alice.connect)(bob.getAddress()) const remoteBob = await p(alice.connect)(bob.getAddress())
assert('alice connected to bob') assert('alice connected to bob')
@ -357,21 +357,21 @@ test('sync a thread with reactions too', async (t) => {
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A1' }, data: { text: 'A1' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
const replyA2 = await p(alice.db.feed.publish)({ const replyA2 = await p(alice.db.feed.publish)({
account: aliceID, account: aliceID,
domain: 'post', domain: 'post',
data: { text: 'A2' }, data: { text: 'A2' },
tangles: [rootA.hash], tangles: [rootA.id],
}) })
const reactionA3 = await p(alice.db.feed.publish)({ const reactionA3 = await p(alice.db.feed.publish)({
account: aliceID, account: aliceID,
domain: 'reaction', domain: 'reaction',
data: { text: 'yes', link: replyA1.hash }, data: { text: 'yes', link: replyA1.id },
tangles: [rootA.hash, replyA1.hash], tangles: [rootA.id, replyA1.id],
}) })
assert.deepEqual( assert.deepEqual(
@ -382,8 +382,8 @@ test('sync a thread with reactions too', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing') assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all') bob.tangleSync.setGoal(rootA.id, 'all')
alice.tangleSync.setGoal(rootA.hash, 'all') alice.tangleSync.setGoal(rootA.id, 'all')
const remoteBob = await p(alice.connect)(bob.getAddress()) const remoteBob = await p(alice.connect)(bob.getAddress())
assert('alice connected to bob') assert('alice connected to bob')