diff --git a/.gitignore b/.gitignore index 3a1f1fd..aec2bbe 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ node_modules pnpm-lock.yaml package-lock.json coverage -**/*.d.ts +lib/**/*.d.ts *~ # For misc scripts and experiments: diff --git a/declarations/atomic-file-rw.d.ts b/declarations/atomic-file-rw.d.ts new file mode 100644 index 0000000..2e801e8 --- /dev/null +++ b/declarations/atomic-file-rw.d.ts @@ -0,0 +1,16 @@ +type CB = (...args: [NodeJS.ErrnoException] | [null, T]) => void + +declare module 'atomic-file-rw' { + export function readFile( + path: string, + encodingOrOpts: string | { encoding: string }, + cb: CB + ): void + export function writeFile( + path: string, + data: string, + encodingOrOpts: string | { encoding: string }, + cb: CB + ): void + export function deleteFile(path: string, cb: CB): void +} diff --git a/declarations/multicb.d.ts b/declarations/multicb.d.ts new file mode 100644 index 0000000..7251fbe --- /dev/null +++ b/declarations/multicb.d.ts @@ -0,0 +1,10 @@ +declare module 'multicb' { + type Opts = { + pluck?: number + spread?: boolean + } + type CB = (...args: [Error] | [null, T] | []) => void + type Done = ((cb: CB) => void) & (() => CB) + function multicb(opts?: Opts): Done + export = multicb +} diff --git a/declarations/mutexify.d.ts b/declarations/mutexify.d.ts new file mode 100644 index 0000000..2e8b6e4 --- /dev/null +++ b/declarations/mutexify.d.ts @@ -0,0 +1,12 @@ +declare module 'mutexify' { + type CB = T extends void + ? (...args: [NodeJS.ErrnoException] | []) => void + : (...args: [NodeJS.ErrnoException] | [null, T]) => void + export type Mutexify = ( + fn: ( + unlock: (cb: CB, ...args: [Error] | [null, T]) => void + ) => void + ) => void + function mutexify(): Mutexify + export = mutexify +} diff --git a/declarations/obz.d.ts b/declarations/obz.d.ts new file mode 100644 index 0000000..6795b43 --- /dev/null +++ b/declarations/obz.d.ts @@ -0,0 +1,10 @@ +declare module 'obz' { + type Remove = () => void + export interface Obz { + (listener: (value: X) => void): Remove + set(value: X): this + value: X + } + function createObz(): Obz + export = createObz +} diff --git a/lib/ghosts.js b/lib/ghosts.js index e544b3d..5157a7b 100644 --- a/lib/ghosts.js +++ b/lib/ghosts.js @@ -1,16 +1,22 @@ const FS = require('fs') const Path = require('path') -// @ts-ignore const atomic = require('atomic-file-rw') -// @ts-ignore const multicb = require('multicb') -// @ts-ignore const mutexify = require('mutexify') const ReadyGate = require('./utils/ready-gate') // TODO: fs is only supported in node.js. We should support browser by replacing // fs.readdir with a browser "file" that just lists all ghost files. +/** + * @typedef {import('./index').MsgID} MsgID + */ + +/** + * @template T + * @typedef {import('mutexify').Mutexify} Mutexify + */ + /** * @template T * @typedef {T extends void ? @@ -22,14 +28,11 @@ const ReadyGate = require('./utils/ready-gate') class Ghosts { /** @type {string} */ #basePath - /** @type {ReadyGate} */ #loaded - - /** @type {Map>} */ + /** @type {Map>} */ #maps - - /** @type {(fn: (unlock: (cb: CB, ...args: ([Error] | [null, null])) => void) => void) => void} */ + /** @type {Mutexify} */ #writeLock static encodingOpts = { encoding: 'utf-8' } @@ -58,7 +61,7 @@ class Ghosts { cb() }) }) - done((/** @type {any} */ err) => { + done((err, _) => { // prettier-ignore if (err) throw new Error('GhostDB failed to load', { cause: err }) this.#loaded.setReady() @@ -96,21 +99,17 @@ class Ghosts { * @param {CB>} cb */ #read(tangleID, cb) { - atomic.readFile( - this.#path(tangleID), - Ghosts.encodingOpts, - (/** @type {any} */ err, /** @type {any} */ str) => { - // Load Map - /** @type {Map} */ - let map - if (err && err.code === 'ENOENT') map = new Map() + atomic.readFile(this.#path(tangleID), Ghosts.encodingOpts, (err, str) => { + // Load Map + /** @type {Map} */ + let map + if (err && err.code === 'ENOENT') map = new Map() // prettier-ignore else if (err) return cb(new Error('GhostDB.read() failed to read ghost file', { cause: err })) else map = this.#deserialize(str) - cb(null, map) - } - ) + cb(null, map) + }) } /** @@ -148,11 +147,11 @@ class Ghosts { this.#path(tangleID), this.#serialize(newMap), Ghosts.encodingOpts, - (/** @type {any} */ err) => { + (err, _) => { // prettier-ignore if (err) return unlock(cb, new Error('GhostDB.save() failed to write ghost file', { cause: err })) this.#maps.set(tangleID, newMap) - unlock(cb, null, null) + unlock(cb, null, void 0) } ) }) @@ -167,12 +166,12 @@ class Ghosts { remove(tangleID, msgID, cb) { this.#writeLock((unlock) => { this.#loaded.onReady(() => { - if (!this.#maps.has(tangleID)) return unlock(cb, null, null) + if (!this.#maps.has(tangleID)) return unlock(cb, null, void 0) const map = /** @type {Map} */ ( this.#maps.get(tangleID) ) - if (!map.has(msgID)) return unlock(cb, null, null) + if (!map.has(msgID)) return unlock(cb, null, void 0) const newMap = new Map(map) newMap.delete(msgID) @@ -181,11 +180,11 @@ class Ghosts { this.#path(tangleID), this.#serialize(newMap), Ghosts.encodingOpts, - (/** @type {any} */ err) => { + (err, _) => { // prettier-ignore if (err) return unlock(cb,new Error('GhostDB.save() failed to write ghost file', { cause: err })) this.#maps.set(tangleID, newMap) - unlock(cb, null, null) + unlock(cb, null, void 0) } ) }) diff --git a/lib/index.js b/lib/index.js index 6f11b8e..557397e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,7 +2,6 @@ const Path = require('path') const promisify = require('promisify-4loc') const b4a = require('b4a') const base58 = require('bs58') -// @ts-ignore const Obz = require('obz') const Keypair = require('ppppp-keypair') const Log = require('./log') @@ -30,6 +29,12 @@ const { decrypt } = require('./encryption') * @typedef {Buffer | Uint8Array} B4A * @typedef {{global: {keypair: Keypair; path: string}}} ExpectedConfig * @typedef {{global: {keypair: Keypair; path?: string}}} Config + * @typedef {{ + * close: { + * (errOrEnd: boolean, cb?: CB): void, + * hook(hookIt: (this: unknown, fn: any, args: any) => any): void + * }; + * }} Peer */ /** @@ -71,6 +76,11 @@ const { decrypt } = require('./encryption') * } CB */ +/** + * @template T + * @typedef {import('obz').Obz} Obz + */ + /** * @param {Config} config * @returns {asserts config is ExpectedConfig} @@ -173,7 +183,7 @@ class DBTangle extends MsgV4.Tangle { } /** - * @param {any} peer + * @param {Peer} peer * @param {Config} config */ function initDB(peer, config) { @@ -181,13 +191,11 @@ function initDB(peer, config) { /** @type {Array} */ const recs = [] - /** @type {WeakMap} */ const miscRegistry = new WeakMap() - /** @type {Map} */ const encryptionFormats = new Map() - + /** @type {Obz} */ const onRecordAdded = Obz() const codec = { @@ -225,9 +233,8 @@ function initDB(peer, config) { const ghosts = new Ghosts(Path.join(config.global.path, 'db', 'ghosts')) - peer.close.hook(function (/** @type {any} */ fn, /** @type {any} */ args) { + peer.close.hook(function hookToCloseDB(fn, args) { log.close(() => { - // @ts-ignore fn.apply(this, args) }) }) diff --git a/lib/log/errors.js b/lib/log/errors.js index 83d26d7..c48e8c6 100644 --- a/lib/log/errors.js +++ b/lib/log/errors.js @@ -13,20 +13,14 @@ class ErrorWithCode extends Error { * @param {number} offset */ function nanOffsetErr(offset) { - return new ErrorWithCode( - `Offset ${offset} is not a number`, - 'ERR_AAOL_INVALID_OFFSET' - ) + return new ErrorWithCode(`Offset ${offset} is not a number`, 'INVALID_OFFSET') } /** * @param {number} offset */ function negativeOffsetErr(offset) { - return new ErrorWithCode( - `Offset ${offset} is negative`, - 'ERR_AAOL_INVALID_OFFSET' - ) + return new ErrorWithCode(`Offset ${offset} is negative`, 'INVALID_OFFSET') } /** @@ -36,12 +30,12 @@ function negativeOffsetErr(offset) { function outOfBoundsOffsetErr(offset, logSize) { return new ErrorWithCode( `Offset ${offset} is beyond log size ${logSize}`, - 'ERR_AAOL_OFFSET_OUT_OF_BOUNDS' + 'OFFSET_OUT_OF_BOUNDS' ) } function deletedRecordErr() { - return new ErrorWithCode('Record has been deleted', 'ERR_AAOL_DELETED_RECORD') + return new ErrorWithCode('Record has been deleted', 'DELETED_RECORD') } function delDuringCompactErr() { diff --git a/lib/log/index.js b/lib/log/index.js index 4c0d04c..7c69e5d 100644 --- a/lib/log/index.js +++ b/lib/log/index.js @@ -1,14 +1,14 @@ const fs = require('fs') const b4a = require('b4a') const p = require('promisify-tuple') +const AtomicFile = require('atomic-file-rw') +const mutexify = require('mutexify') +const Obz = require('obz') // @ts-ignore const Cache = require('@alloc/quick-lru') // @ts-ignore const RAF = require('polyraf') // @ts-ignore -const Obv = require('obz') // @ts-ignore -const AtomicFile = require('atomic-file-rw') // @ts-ignore const debounce = require('lodash.debounce') // @ts-ignore const isBufferZero = require('is-buffer-zero') // @ts-ignore -const debug = require('debug')('ppppp-db:log') // @ts-ignore -const mutexify = require('mutexify') +const debug = require('debug')('ppppp-db:log') const { deletedRecordErr, @@ -26,6 +26,16 @@ const Record = require('./record') * @typedef {number} BlockIndex */ +/** + * @template T + * @typedef {import('mutexify').Mutexify} Mutexify + */ + +/** + * @template T + * @typedef {import('obz').Obz} Obz + */ + /** * @template T * @typedef {{ @@ -120,63 +130,60 @@ function Log(filename, opts) { let latestBlockIndex = /** @type {number | null} */ (null) let nextOffsetInBlock = /** @type {number | null} */ (null) let deletedBytes = 0 - const lastRecOffset = Obv() // offset of last written record + /** Offset of last written record @type {Obz} */ + const lastRecOffset = Obz() let compacting = false - const compactionProgress = Obv() + const compactionProgress = Obz() compactionProgress.set(COMPACTION_PROGRESS_START) /** @type {Array>} */ const waitingCompaction = [] - AtomicFile.readFile( - statsFilename, - 'utf8', - /** @type {CB} */ function doneLoadingStatsFile(err, json) { - if (err) { + AtomicFile.readFile(statsFilename, 'utf8', function onStatsLoaded(err, json) { + if (err) { + // prettier-ignore + if (err.code !== 'ENOENT') debug('Failed loading stats file: %s', err.message) + deletedBytes = 0 + } else { + try { + const stats = JSON.parse(json) + deletedBytes = stats.deletedBytes + } catch (err) { // prettier-ignore - if (err.code !== 'ENOENT') debug('Failed loading stats file: %s', err.message) + debug('Failed parsing stats file: %s', /** @type {Error} */ (err).message) deletedBytes = 0 - } else { - try { - const stats = JSON.parse(json) - deletedBytes = stats.deletedBytes - } catch (err) { - // prettier-ignore - debug('Failed parsing stats file: %s', /** @type {Error} */ (err).message) - deletedBytes = 0 - } } + } - raf.stat( - /** @type {CB<{size: number}>} */ function onRAFStatDone(err, stat) { - // prettier-ignore - if (err && err.code !== 'ENOENT') debug('Failed to read %s stats: %s', filename, err.message) + raf.stat( + /** @type {CB<{size: number}>} */ function onRAFStatDone(err, stat) { + // prettier-ignore + if (err && err.code !== 'ENOENT') debug('Failed to read %s stats: %s', filename, err.message) - const fileSize = stat ? stat.size : -1 + const fileSize = stat ? stat.size : -1 - if (fileSize <= 0) { - debug('Opened log file, which is empty') - latestBlockBuf = b4a.alloc(blockSize) - latestBlockIndex = 0 - nextOffsetInBlock = 0 - cache.set(0, latestBlockBuf) - lastRecOffset.set(-1) + if (fileSize <= 0) { + debug('Opened log file, which is empty') + latestBlockBuf = b4a.alloc(blockSize) + latestBlockIndex = 0 + nextOffsetInBlock = 0 + cache.set(0, latestBlockBuf) + lastRecOffset.set(-1) + // @ts-ignore + while (waitingLoad.length) waitingLoad.shift()() + } else { + const blockStart = fileSize - blockSize + loadLatestBlock(blockStart, function onLoadedLatestBlock(err) { + if (err) throw err + // prettier-ignore + debug('Opened log file, last record is at log offset %d, block %d', lastRecOffset.value, latestBlockIndex) // @ts-ignore while (waitingLoad.length) waitingLoad.shift()() - } else { - const blockStart = fileSize - blockSize - loadLatestBlock(blockStart, function onLoadedLatestBlock(err) { - if (err) throw err - // prettier-ignore - debug('Opened log file, last record is at log offset %d, block %d', lastRecOffset.value, latestBlockIndex) - // @ts-ignore - while (waitingLoad.length) waitingLoad.shift()() - }) - } + }) } - ) - } - ) + } + ) + }) /** * @param {number} blockStart @@ -236,7 +243,7 @@ function Log(filename, opts) { return getBlockStart(offset) / blockSize } - /** @type {(fn: (unlock: (cb: CB, ...args: ([Error] | [null, any])) => void) => void) => void} */ + /** @type {Mutexify} */ const writeLock = mutexify() /** @@ -697,7 +704,10 @@ function Log(filename, opts) { */ function saveStats(cb) { const stats = JSON.stringify({ deletedBytes }) - AtomicFile.writeFile(statsFilename, stats, 'utf8', cb) + AtomicFile.writeFile(statsFilename, stats, 'utf8', (err, _) => { + if (err) return cb(new Error('Failed to save stats file', { cause: err })) + cb() + }) } /** @type {CB} */ diff --git a/test/log/delete.test.js b/test/log/delete.test.js index 7505280..6bcefa4 100644 --- a/test/log/delete.test.js +++ b/test/log/delete.test.js @@ -41,7 +41,7 @@ test('Log deletes', async (t) => { await assert.rejects(p(log._get)(offset2), (err) => { assert.ok(err) assert.equal(err.message, 'Record has been deleted') - assert.equal(err.code, 'ERR_AAOL_DELETED_RECORD') + assert.equal(err.code, 'DELETED_RECORD') return true }) @@ -103,7 +103,7 @@ test('Log deletes', async (t) => { await assert.rejects(p(log2._get)(offset2), (err) => { assert.ok(err) assert.equal(err.message, 'Record has been deleted') - assert.equal(err.code, 'ERR_AAOL_DELETED_RECORD') + assert.equal(err.code, 'DELETED_RECORD') return true }) diff --git a/tsconfig.json b/tsconfig.json index cc6e7af..7bd7313 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,5 @@ { - "include": ["lib/**/*.js"], + "include": ["declarations", "lib/**/*.js"], "exclude": ["coverage/", "node_modules/", "test/"], "compilerOptions": { "checkJs": true, @@ -11,6 +11,7 @@ "module": "node16", "skipLibCheck": true, "strict": true, - "target": "es2022" + "target": "es2022", + "typeRoots": ["node_modules/@types", "declarations"] } } \ No newline at end of file