pzp
/
pzp-sync
mirror of https://codeberg.org/pzp/pzp-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update to new ppppp-db with rec.id

This commit is contained in:
Andre Staltz 2023-09-07 17:31:11 +03:00
parent f6f16bdd47
commit 317f7c0a05
No known key found for this signature in database
GPG Key ID: 9EDE23EA7E8A4890
4 changed files with 95 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
lib/algorithm.js
View File

@ -25,18 +25,18 @@ class Algorithm {
this.#peer = peer
}
haveRange(rootMsgHash) {
const rootMsg = this.#peer.db.get(rootMsgHash)
haveRange(rootID) {
const rootMsg = this.#peer.db.get(rootID)
if (!rootMsg) return [1, 0]
let minDepth = Number.MAX_SAFE_INTEGER
let maxDepth = 0
for (const rec of this.#peer.db.records()) {
if (!rec.msg?.data) continue
if (!rec?.msg?.data) continue
const tangles = rec.msg.metadata.tangles
if (rec.hash === rootMsgHash) {
if (rec.id === rootID) {
minDepth = 0
} else if (tangles[rootMsgHash]) {
const depth = tangles[rootMsgHash].depth
} else if (tangles[rootID]) {
const depth = tangles[rootID].depth
minDepth = Math.min(minDepth, depth)
maxDepth = Math.max(maxDepth, depth)
}
@ -99,13 +99,13 @@ class Algorithm {
}
}
bloomFor(rootMsgHash, round, range, extraIds = []) {
bloomFor(rootID, round, range, extraIds = []) {
const filterSize =
(isEmptyRange(range) ? 2 : estimateMsgCount(range)) + countIter(extraIds)
const filter = BloomFilter.create(2 * filterSize, 0.00001)
if (!isEmptyRange(range)) {
for (const msg of this.yieldMsgsIn(rootMsgHash, range)) {
filter.add('' + round + MsgV3.getMsgHash(msg))
for (const msg of this.yieldMsgsIn(rootID, range)) {
filter.add('' + round + MsgV3.getMsgID(msg))
}
}
for (const msgId of extraIds) {
@ -114,59 +114,59 @@ class Algorithm {
return filter.saveAsJSON()
}
msgsMissing(rootMsgHash, round, range, remoteBloomJSON) {
msgsMissing(rootID, round, range, remoteBloomJSON) {
if (isEmptyRange(range)) return []
const remoteFilter = BloomFilter.fromJSON(remoteBloomJSON)
const missing = []
for (const msg of this.yieldMsgsIn(rootMsgHash, range)) {
const msgHash = MsgV3.getMsgHash(msg)
if (!remoteFilter.has('' + round + msgHash)) {
missing.push(msgHash)
for (const msg of this.yieldMsgsIn(rootID, range)) {
const msgID = MsgV3.getMsgID(msg)
if (!remoteFilter.has('' + round + msgID)) {
missing.push(msgID)
}
}
return missing
}
*yieldMsgsIn(rootMsgHash, range) {
*yieldMsgsIn(rootID, range) {
const [minDepth, maxDepth] = range
const rootMsg = this.#peer.db.get(rootMsgHash)
const rootMsg = this.#peer.db.get(rootID)
if (!rootMsg) return
if (minDepth === 0) yield rootMsg
for (const msg of this.#peer.db.msgs()) {
const tangles = msg.metadata.tangles
if (
tangles[rootMsgHash] &&
tangles[rootMsgHash].depth >= minDepth &&
tangles[rootMsgHash].depth <= maxDepth
tangles[rootID] &&
tangles[rootID].depth >= minDepth &&
tangles[rootID].depth <= maxDepth
) {
yield msg
}
}
}
async pruneNewest(rootMsgHash, count) {
const tangle = this.#peer.db.getTangle(rootMsgHash)
async pruneNewest(rootID, count) {
const tangle = this.#peer.db.getTangle(rootID)
const sorted = tangle.topoSort()
if (sorted.length <= count) return
const msgHash = sorted[sorted.length - count]
const { deletables, erasables } = tangle.getDeletablesAndErasables(msgHash)
const msgID = sorted[sorted.length - count]
const { deletables, erasables } = tangle.getDeletablesAndErasables(msgID)
const del = p(this.#peer.db.del)
const erase = p(this.#peer.db.erase)
for (const msgHash of deletables) {
await del(msgHash)
for (const msgID of deletables) {
await del(msgID)
}
for (const msgHash of erasables) {
await erase(msgHash)
for (const msgID of erasables) {
await erase(msgID)
}
}
async commit(rootMsgHash, newMsgs, goal, myWantRange) {
async commit(rootID, newMsgs, goal, myWantRange) {
// Filter out dataful newMsgs that are not in my want-range
const [minWant, maxWant] = myWantRange
const validNewMsgs = newMsgs
.filter((msg) => {
const depth = msg.metadata.tangles[rootMsgHash]?.depth ?? 0
if (depth === 0 && MsgV3.getMsgHash(msg) !== rootMsgHash) {
const depth = msg.metadata.tangles[rootID]?.depth ?? 0
if (depth === 0 && MsgV3.getMsgID(msg) !== rootID) {
return false // the rootMsg is the only acceptable depth-zero msg
}
if (!msg.data) {
@ -176,8 +176,8 @@ class Algorithm {
}
})
.sort((a, b) => {
const aDepth = a.metadata.tangles[rootMsgHash]?.depth ?? 0
const bDepth = b.metadata.tangles[rootMsgHash]?.depth ?? 0
const aDepth = a.metadata.tangles[rootID]?.depth ?? 0
const bDepth = b.metadata.tangles[rootID]?.depth ?? 0
return aDepth - bDepth
})
@ -187,39 +187,39 @@ class Algorithm {
// TODO: optimize perf, avoiding await / try / catch
for (const msg of validNewMsgs) {
try {
await p(this.#peer.db.add)(msg, rootMsgHash)
await p(this.#peer.db.add)(msg, rootID)
} catch {}
}
// Prune. Ideally this should be in a garbage collection module
const { type, count } = parseGoal(goal)
if (type === 'newest') return await this.pruneNewest(rootMsgHash, count)
if (type === 'newest') return await this.pruneNewest(rootID, count)
if (type === 'oldest') throw new Error('not implemented') // TODO:
}
/**
* @param {string} rootMsgHash
* @param {Set<string>} msgHashes
* @param {string} rootID
* @param {Set<string>} msgIDs
* @returns
*/
getTangleSlice(rootMsgHash, msgHashes) {
if (msgHashes.size === 0) return []
const tangle = this.#peer.db.getTangle(rootMsgHash)
getTangleSlice(rootID, msgIDs) {
if (msgIDs.size === 0) return []
const tangle = this.#peer.db.getTangle(rootID)
const sorted = tangle.topoSort()
let oldestMsgHash = null
for (const msgHash of sorted) {
if (msgHashes.has(msgHash)) {
oldestMsgHash = msgHash
let oldestMsgID = null
for (const msgID of sorted) {
if (msgIDs.has(msgID)) {
oldestMsgID = msgID
break
}
}
const { erasables } = tangle.getDeletablesAndErasables(oldestMsgHash)
const { erasables } = tangle.getDeletablesAndErasables(oldestMsgID)
const msgs = []
for (const msgHash of sorted) {
let isErasable = erasables.includes(msgHash)
if (!msgHashes.has(msgHash) && !isErasable) continue
const msg = this.#peer.db.get(msgHash)
for (const msgID of sorted) {
let isErasable = erasables.includes(msgID)
if (!msgIDs.has(msgID) && !isErasable) continue
const msg = this.#peer.db.get(msgID)
if (!msg) continue
if (isErasable) {
msgs.push({ ...msg, data: null })

4
test/account-sync.test.js
View File

@ -9,8 +9,8 @@ const bobKeys = Keypair.generate('ed25519', 'bob')
function getAccount(iter) {
return [...iter]
.filter((msg) => msg.metadata.account === 'self' && msg.data)
.map((msg) => msg.data.add.key.bytes)
.filter((m) => m.metadata.account === 'self' && m.data?.action === 'add')
.map((m) => m.data.add.key.bytes)
}
test('sync an account tangle', async (t) => {

46
test/feed-sync.test.js
View File

@ -19,10 +19,10 @@ test('sync a feed with goal=all', async (t) => {
domain: 'account',
_nonce: 'carol',
})
const carolIDMsg = alice.db.get(carolID)
const carolAccountRoot = alice.db.get(carolID)
// Bob knows Carol
await p(bob.db.add)(carolIDMsg, carolID)
await p(bob.db.add)(carolAccountRoot, carolID)
const carolMsgs = []
for (let i = 1; i <= 10; i++) {
@ -36,12 +36,12 @@ test('sync a feed with goal=all', async (t) => {
}
assert('alice has msgs 1..10 from carol')
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash)
const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsMoot = alice.db.get(carolPostsMootID)
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash)
await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 7; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash)
await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
}
{
@ -55,8 +55,8 @@ test('sync a feed with goal=all', async (t) => {
)
}
bob.tangleSync.setGoal(carolPostsRootHash, 'all')
alice.tangleSync.setGoal(carolPostsRootHash, 'all')
bob.tangleSync.setGoal(carolPostsMootID, 'all')
alice.tangleSync.setGoal(carolPostsMootID, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice')
@ -93,10 +93,10 @@ test('sync a feed with goal=newest', async (t) => {
domain: 'account',
_nonce: 'carol',
})
const carolIDMsg = alice.db.get(carolID)
const carolAccountRoot = alice.db.get(carolID)
// Bob knows Carol
await p(bob.db.add)(carolIDMsg, carolID)
await p(bob.db.add)(carolAccountRoot, carolID)
const carolMsgs = []
for (let i = 1; i <= 10; i++) {
@ -110,12 +110,12 @@ test('sync a feed with goal=newest', async (t) => {
}
assert('alice has msgs 1..10 from carol')
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash)
const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsMoot = alice.db.get(carolPostsMootID)
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash)
await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 7; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash)
await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
}
{
@ -129,8 +129,8 @@ test('sync a feed with goal=newest', async (t) => {
)
}
bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5')
alice.tangleSync.setGoal(carolPostsRootHash, 'all')
bob.tangleSync.setGoal(carolPostsMootID, 'newest-5')
alice.tangleSync.setGoal(carolPostsMootID, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice')
@ -183,11 +183,11 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
carolMsgs.push(rec.msg)
}
const carolPostsRootHash = alice.db.feed.getId(carolID, 'post')
const carolPostsRootMsg = alice.db.get(carolPostsRootHash)
const carolPostsMootID = alice.db.feed.getID(carolID, 'post')
const carolPostsMoot = alice.db.get(carolPostsMootID)
const algo = new Algorithm(alice)
await algo.pruneNewest(carolPostsRootHash, 5)
await algo.pruneNewest(carolPostsMootID, 5)
{
const arr = [...alice.db.msgs()]
.filter((msg) => msg.metadata.account === carolID && msg.data)
@ -199,9 +199,9 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
)
}
await p(bob.db.add)(carolPostsRootMsg, carolPostsRootHash)
await p(bob.db.add)(carolPostsMoot, carolPostsMootID)
for (let i = 0; i < 2; i++) {
await p(bob.db.add)(carolMsgs[i], carolPostsRootHash)
await p(bob.db.add)(carolMsgs[i], carolPostsMootID)
}
{
@ -211,8 +211,8 @@ test('sync a feed with goal=newest but too far behind', async (t) => {
assert.deepEqual(arr, ['m1', 'm2'], 'bob has msgs 1..2 from carol')
}
alice.tangleSync.setGoal(carolPostsRootHash, 'newest-5')
bob.tangleSync.setGoal(carolPostsRootHash, 'newest-5')
alice.tangleSync.setGoal(carolPostsMootID, 'newest-5')
bob.tangleSync.setGoal(carolPostsMootID, 'newest-5')
const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice')

44
test/thread-sync.test.js
View File

@ -95,7 +95,7 @@ test('sync a thread where both peers have portions', async (t) => {
domain: 'post',
data: { text: 'A' },
})
const rootHashA = alice.db.feed.getId(aliceID, 'post')
const rootHashA = alice.db.feed.getID(aliceID, 'post')
const rootMsgA = alice.db.get(rootHashA)
await p(bob.db.add)(rootMsgA, rootHashA)
@ -105,16 +105,16 @@ test('sync a thread where both peers have portions', async (t) => {
account: bobID,
domain: 'post',
data: { text: 'B1' },
tangles: [startA.hash],
tangles: [startA.id],
})
const replyB2 = await p(bob.db.feed.publish)({
account: bobID,
domain: 'post',
data: { text: 'B2' },
tangles: [startA.hash],
tangles: [startA.id],
})
const rootHashB = bob.db.feed.getId(bobID, 'post')
const rootHashB = bob.db.feed.getID(bobID, 'post')
const rootMsgB = bob.db.get(rootHashB)
await p(alice.db.add)(rootMsgB, rootHashB)
@ -125,7 +125,7 @@ test('sync a thread where both peers have portions', async (t) => {
account: carolID,
domain: 'post',
data: { text: 'C1' },
tangles: [startA.hash],
tangles: [startA.id],
keypair: carolKeypair,
})
@ -133,7 +133,7 @@ test('sync a thread where both peers have portions', async (t) => {
account: daveID,
domain: 'post',
data: { text: 'D1' },
tangles: [startA.hash],
tangles: [startA.id],
keypair: daveKeypair,
})
@ -149,8 +149,8 @@ test('sync a thread where both peers have portions', async (t) => {
'bob has another portion of the thread'
)
bob.tangleSync.setGoal(startA.hash, 'all')
alice.tangleSync.setGoal(startA.hash, 'all')
bob.tangleSync.setGoal(startA.id, 'all')
alice.tangleSync.setGoal(startA.id, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice')
@ -210,14 +210,14 @@ test('sync a thread where initiator does not have the root', async (t) => {
account: aliceID,
domain: 'post',
data: { text: 'A1' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
const replyA2 = await p(alice.db.feed.publish)({
account: aliceID,
domain: 'post',
data: { text: 'A2' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
assert.deepEqual(
@ -228,9 +228,9 @@ test('sync a thread where initiator does not have the root', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all')
bob.tangleSync.setGoal(rootA.id, 'all')
// ON PURPOSE: alice does not set the goal
// alice.tangleSync.setGoal(rootA.hash, 'all')
// alice.tangleSync.setGoal(rootA.id, 'all')
const remoteAlice = await p(bob.connect)(alice.getAddress())
assert('bob connected to alice')
@ -284,14 +284,14 @@ test('sync a thread where receiver does not have the root', async (t) => {
account: aliceID,
domain: 'post',
data: { text: 'A1' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
const replyA2 = await p(alice.db.feed.publish)({
account: aliceID,
domain: 'post',
data: { text: 'A2' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
assert.deepEqual(
@ -302,8 +302,8 @@ test('sync a thread where receiver does not have the root', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all')
alice.tangleSync.setGoal(rootA.hash, 'all')
bob.tangleSync.setGoal(rootA.id, 'all')
alice.tangleSync.setGoal(rootA.id, 'all')
const remoteBob = await p(alice.connect)(bob.getAddress())
assert('alice connected to bob')
@ -357,21 +357,21 @@ test('sync a thread with reactions too', async (t) => {
account: aliceID,
domain: 'post',
data: { text: 'A1' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
const replyA2 = await p(alice.db.feed.publish)({
account: aliceID,
domain: 'post',
data: { text: 'A2' },
tangles: [rootA.hash],
tangles: [rootA.id],
})
const reactionA3 = await p(alice.db.feed.publish)({
account: aliceID,
domain: 'reaction',
data: { text: 'yes', link: replyA1.hash },
tangles: [rootA.hash, replyA1.hash],
data: { text: 'yes', link: replyA1.id },
tangles: [rootA.id, replyA1.id],
})
assert.deepEqual(
@ -382,8 +382,8 @@ test('sync a thread with reactions too', async (t) => {
assert.deepEqual(getTexts(bob.db.msgs()), [], 'bob has nothing')
bob.tangleSync.setGoal(rootA.hash, 'all')
alice.tangleSync.setGoal(rootA.hash, 'all')
bob.tangleSync.setGoal(rootA.id, 'all')
alice.tangleSync.setGoal(rootA.id, 'all')
const remoteBob = await p(alice.connect)(bob.getAddress())
assert('alice connected to bob')