mpackdb

27.0 KB

1. import { dirname, basename, extname, resolve } from 'path';
2. import { createReadStream, createWriteStream } from 'fs';
3. import { mkdir, open, stat, readFile, writeFile, rename, unlink } from 'fs/promises';
4. import { serialize, deserialize, uuid, PrimaryKeyType, IndexType } from './mpack.js';
5. import { Cursor } from './Cursor.js';
6. import { IndexManager } from './IndexManager.js';
8. export { PrimaryKeyType, IndexType };
10. /**
11. * MPackDB - A fast, local, append-only JSON database with MessagePack serialization
12. *
13. * Features:
14. * - Append-only writes for high performance
15. * - MessagePack binary serialization
16. * - Optional indexes (numeric and lexical)
17. * - File-based locking for concurrent access
18. * - Auto-compaction on startup
19. * - Auto-persistence of indexes
20. */
21. export class MPackDB {
22. _dbFile = null;
23. _primaryKeyType = null;
24. _primaryKey = null;
25. _classToUse = null;
27. _indexes = [];
28. _indexTypes = {};
29. _uniqueIndexes = new Set();
31. _initPromise = null;
32. _initState = 0;
33. _dataPath = null;
34. _dataStream = null;
35. _lockPath = null;
36. _lockDepth = 0;
37. _indexManager = null;
38. _meta = {
39. nextId: 0,
40. deleted: [],
41. };
42. _debug = false;
43. _indexPersistInterval = 60000; // 60 seconds default
44. _indexPersistThreshold = 1000; // 1000 entries default
45. _processExitHandler = null;
47. /**
48. * Create a new MPackDB instance
49. *
50. * @param {string} dbFile - Path to the database file (without extension)
51. * @param {Object} options - Configuration options
52. * @param {string} [options.primaryKey] - Primary key field name. Prefix with * for numeric (e.g., '*id'), @ for UUID (e.g., '@uuid'), or no prefix for string
53. * @param {PrimaryKeyType} [options.primaryKeyType] - Explicit primary key type (overrides prefix)
54. * @param {string[]} [options.indexes] - Array of field names to index. Use * prefix for numeric, @ for UUID
55. * @param {boolean} [options.debug=false] - Enable debug logging
56. * @param {number} [options.indexPersistInterval=60000] - Milliseconds between automatic index persistence (0 to disable)
57. * @param {number} [options.indexPersistThreshold=1000] - Number of changes before auto-persisting indexes
58. * @param {boolean} [options.compact=true] - Run compaction on init (set false for read-only / secondary instances)
59. *
60. * @example
61. * const db = new MPackDB('data/users', {
62. * primaryKey: '*id', // Numeric auto-increment
63. * indexes: ['email', '*age'], // Index email (lexical) and age (numeric)
64. * indexPersistThreshold: 100
65. * });
66. */
67. _compact = true;
69. constructor(dbFile, { primaryKey, primaryKeyType, indexes, debug, indexPersistInterval, indexPersistThreshold, compact } = {}) {
70. this._dbFile = dbFile || this._dbFile;
71. this._debug = debug || this._debug;
72. if (compact !== undefined) this._compact = compact;
73. if (indexPersistInterval !== undefined) this._indexPersistInterval = indexPersistInterval;
74. if (indexPersistThreshold !== undefined) this._indexPersistThreshold = indexPersistThreshold;
76. // Parse primary key with optional prefix
77. if (primaryKey) {
78. if (primaryKey.startsWith('*')) {
79. // *id = numeric primary key
80. this._primaryKey = primaryKey.slice(1);
81. this._primaryKeyType = PrimaryKeyType.NUMBER;
82. } else if (primaryKey.startsWith('@')) {
83. // @id = UUID primary key
84. this._primaryKey = primaryKey.slice(1);
85. this._primaryKeyType = PrimaryKeyType.UUID;
86. } else {
87. // id = string primary key (lexical)
88. this._primaryKey = primaryKey;
89. this._primaryKeyType = PrimaryKeyType.STRING;
90. }
91. // Allow explicit override
92. if (primaryKeyType !== undefined) {
93. this._primaryKeyType = primaryKeyType;
94. }
95. }
97. // Primary key is always unique
98. if (this._primaryKey) {
99. this._uniqueIndexes.add(this._primaryKey);
100. }
102. const allIndexes = [...new Set([
103. ...(this._primaryKey ? [this._primaryKey] : []),
104. ...(indexes || []),
105. ...(this._indexes || []),
106. ])];
107. this._indexes = [];
108. for (let index of allIndexes) {
109. let cleanIndex = index;
110. // !field = unique index (can combine with * and @: !*field, !@field)
111. let isUnique = false;
112. if (cleanIndex.startsWith('!')) {
113. isUnique = true;
114. cleanIndex = cleanIndex.slice(1);
115. }
116. if (cleanIndex.startsWith('*')) {
117. // *field = numeric index
118. cleanIndex = cleanIndex.slice(1);
119. this._indexTypes[cleanIndex] = IndexType.NUMERIC;
120. } else if (cleanIndex.startsWith('@')) {
121. // @field = UUID index (lexical)
122. cleanIndex = cleanIndex.slice(1);
123. this._indexTypes[cleanIndex] = IndexType.LEXICAL;
124. } else if (cleanIndex === this._primaryKey && this._primaryKeyType === PrimaryKeyType.NUMBER) {
125. // Primary key is numeric
126. this._indexTypes[cleanIndex] = IndexType.NUMERIC;
127. } else {
128. // Default to lexical
129. this._indexTypes[cleanIndex] = IndexType.LEXICAL;
130. }
131. if (isUnique) {
132. this._uniqueIndexes.add(cleanIndex);
133. }
134. this._indexes.push(cleanIndex);
135. }
136. }
138. /**
139. * Initialize the database (called automatically by other methods)
140. * Performs compaction and loads metadata
141. *
142. * @returns {Promise<MPackDB>} The database instance
143. */
144. async init() {
145. if (this._initState === 2) return this;
147. if (this._initState === 1) {
148. return this._initPromise;
149. }
151. this._initState = 1;
152. return this._initPromise = new Promise(async success => {
153. if (!this._dbFile) {
154. throw new Error('No database file specified.');
155. }
157. const dbDir = dirname(this._dbFile);
158. const baseName = basename(this._dbFile, extname(this._dbFile));
159. await mkdir(dbDir, { recursive: true });
161. this._dataPath = resolve(dbDir, `${baseName}.mpack`);
162. this._metaPath = resolve(dbDir, `${baseName}.meta.json`);
163. this._lockPath = resolve(dbDir, `${baseName}.lock`);
165. try {
166. this._meta = JSON.parse(await readFile(this._metaPath));
167. } catch (e) {
168. this._meta = {
169. nextId: 0,
170. deleted: [],
171. };
172. }
174. this._initState = 2; // compact causes init to run again thats why we set it to 2 here already
176. if (this._compact) await this.compact();
178. this._dataStream = createWriteStream(this._dataPath, { flags: 'a' }); // needs to be set after compact as it replaces the file with a tmp file
180. // Initialize IndexManager if indexes are specified
181. if (this._indexes.length > 0) {
182. const dbDir = dirname(this._dbFile);
183. const baseName = basename(this._dbFile, extname(this._dbFile));
184. this._indexManager = new IndexManager(dbDir, baseName, this._indexes, this._indexTypes, this._primaryKeyType);
185. await this._indexManager.init(this._dataPath);
186. // Start auto-persist with configured interval and threshold
187. this._indexManager.startAutoPersist(this._indexPersistInterval, this._indexPersistThreshold);
188. // Register signal handlers for Ctrl+C and kill signals
189. this._processExitHandler = async () => {
190. await this.close();
191. process.exit(0);
192. };
193. process.once('SIGINT', this._processExitHandler);
194. process.once('SIGTERM', this._processExitHandler);
195. }
197. this._initPromise = null;
198. success(this);
199. });
200. }
202. /**
203. * Insert a new record into the database
204. *
205. * @param {Object} record - The record to insert
206. * @param {Object} [options]
207. * @param {boolean} [options.skipPrimaryKey=false] - Skip auto-generating primary key
208. * @returns {Promise<Object>} The inserted record with primary key
209. *
210. * @example
211. * await db.insert({ name: 'Alice', age: 30 });
212. * // Returns: { id: 0, name: 'Alice', age: 30 }
213. */
214. async insert(record, { skipPrimaryKey = false } = {}) {
215. await this.init();
216. await this._acquireLock('insert', record);
217. try {
219. const recToInsert = this._classToUse
220. ? Object.assign(new this._classToUse(), record)
221. : { ...record };
223. if (this._primaryKey && !skipPrimaryKey && !recToInsert[this._primaryKey]) {
224. if (this._primaryKeyType === PrimaryKeyType.NUMBER) {
225. recToInsert[this._primaryKey] = this._meta.nextId++;
226. } else if (this._primaryKeyType === PrimaryKeyType.UUID) {
227. recToInsert[this._primaryKey] = uuid();
228. }
229. }
231. // Check unique index constraints (skip auto-generated primary keys — guaranteed unique)
232. if (this._indexManager && this._uniqueIndexes.size > 0) {
233. const autoGenPK = this._primaryKey && !skipPrimaryKey && !record[this._primaryKey]
234. && (this._primaryKeyType === PrimaryKeyType.NUMBER || this._primaryKeyType === PrimaryKeyType.UUID);
235. const deletedSet = new Set(this._meta.deleted);
236. for (const field of this._uniqueIndexes) {
237. if (autoGenPK && field === this._primaryKey) continue;
238. const value = recToInsert[field];
239. if (value === undefined) continue;
240. const existing = await this._indexManager.get(field, value);
241. const live = existing.filter(e => !deletedSet.has(e.loc[0]));
242. if (live.length > 0) {
243. const err = new Error(`Duplicate key: ${field}=${value}`);
244. err.code = 'DUPLICATE_KEY';
245. err.field = field;
246. err.value = value;
247. throw err;
248. }
249. }
250. }
252. const packedBuffer = serialize(recToInsert);
253. const fileStat = await stat(this._dataPath).catch(() => ({ size: 0 }));
254. const offset = fileStat.size;
256. await new Promise(resolve => this._dataStream.write(packedBuffer, resolve));
258. const loc = [offset, packedBuffer.length];
259. // indexes
260. if (this._indexManager) {
261. this._indexManager.insert(recToInsert, loc);
262. }
264. // Persist meta if we incremented nextId
265. if (this._primaryKey && this._primaryKeyType === PrimaryKeyType.NUMBER && !skipPrimaryKey && !record[this._primaryKey]) {
266. await this.persistMeta();
267. }
269. return this._primaryKey ? recToInsert[this._primaryKey] : recToInsert;
270. } finally {
271. await this._releaseLock();
272. }
273. }
275. /**
276. * Update records matching a query
277. *
278. * @param {string|Function} mixed - Primary key value or query function
279. * @param {Object|Function} dataOrCallback - Data to update or callback function
280. * @param {Object} [options]
281. * @param {boolean} [options.upsert=false] - Insert if no records match
282. * @param {Object|Array} [options.index] - Index hint(s) to narrow disk reads
283. * @returns {Promise<Object[]>} Array of updated records
284. *
285. * @example
286. * // Update by primary key
287. * await db.update(0, { age: 31 });
288. *
289. * // Update with query function
290. * await db.update(r => r.age > 30, { status: 'senior' });
291. *
292. * // Update with callback
293. * await db.update(r => r.age > 30, r => ({ ...r, age: r.age + 1 }));
294. */
295. async update(mixed, dataOrCallback, { upsert = false, index } = {}) {
296. let insertedRecords = await this.delete(mixed, {
297. index,
298. callback: async record => {
299. return this.insert(typeof dataOrCallback === 'function'
300. ? await dataOrCallback(record)
301. : dataOrCallback,
302. { skipPrimaryKey: true }
303. );
304. }
305. });
306. if (upsert && insertedRecords.length === 0) {
307. insertedRecords = await this.insert(typeof dataOrCallback === 'function'
308. ? await dataOrCallback({})
309. : dataOrCallback);
310. }
311. return insertedRecords;
312. }
314. /**
315. * Update records or insert if not found
316. *
317. * @param {string|Function} mixed - Primary key value or query function
318. * @param {Object|Function} dataOrCallback - Data to update/insert or callback function
319. * @param {Object} [options]
320. * @param {Object|Array} [options.index] - Index hint(s) to narrow disk reads
321. * @returns {Promise<Object[]>} Array of updated/inserted records
322. */
323. async upsert(mixed, dataOrCallback, { index } = {}) {
324. return this.update(mixed, dataOrCallback, { upsert: true, index });
325. }
327. /**
328. * Delete records matching a query
329. *
330. * @param {string|Function} mixed - Primary key value or query function
331. * @param {Object} [options]
332. * @param {Object|Array} [options.index] - Index hint(s) to narrow disk reads
333. * @param {Function} [options.callback] - Internal callback per deleted record (used by update)
334. * @returns {Promise<Object[]>} Array of deleted records
335. *
336. * @example
337. * // Delete by primary key
338. * await db.delete(0);
339. *
340. * // Delete with query function
341. * await db.delete(r => r.age < 18);
342. *
343. * // Delete with index hint
344. * await db.delete(r => r.status === 'inactive', { index: { field: 'status', value: 'inactive' } });
345. */
346. async delete(mixed, { index, callback = async record => record } = {}) {
347. await this.init();
348. await this._acquireLock('delete', mixed);
349. try {
350. const promises = [];
351. for await (const [record, offset] of this.find(mixed, { mode: 'mixed', index })) {
352. this._meta.deleted.push(offset);
353. if (this._indexManager) {
354. await this._indexManager.remove(record, this._primaryKey);
355. }
356. promises.push(callback(record));
357. }
358. await this.persistMeta();
359. return Promise.all(promises);
360. } finally {
361. await this._releaseLock();
362. }
363. }
365. /**
366. * Finds records in the database
367. * @param {undefined|string|number|function} [mixed] - Query: undefined/null for all, primary key value for PK lookup, function for filter
368. * @param {Object} [options] - Query options
369. * @param {Object|Array} [options.index] - Index hint(s) to narrow disk reads before filtering
370. * @returns {Cursor} A cursor for iterating over results
371. * @example
372. * // All records
373. * for await (const user of db.find()) { ... }
374. *
375. * // Primary key lookup
376. * const [user] = await db.find(68);
377. *
378. * // Filter with index range — only reads records in x 100+
379. * for await (const doc of db.find(r => r.x <= 200, { index: { field: 'x', from: 100 } })) { ... }
380. *
381. * // Index intersection — intersects offsets first, then streams matches
382. * for await (const doc of db.find(r => r.x <= 200, {
383. * index: [
384. * { field: 'x', from: 100 },
385. * { field: 'status', value: 'active' }
386. * ]
387. * })) { ... }
388. */
389. find(mixed, options = {}) {
390. if (typeof mixed === 'undefined' || mixed === null || typeof mixed === 'function') {
391. return new Cursor(this, mixed, options);
392. } else {
393. if (!this._primaryKey) {
394. throw new Error('No primary key specified.');
395. }
396. // Use indexed lookup when available
397. if (this._indexManager) {
398. return new Cursor(this, null, {
399. ...options,
400. _indexLookup: { field: this._primaryKey, value: mixed }
401. });
402. }
403. return new Cursor(this, record => {
404. return record[this._primaryKey] === mixed;
405. }, options);
406. }
407. }
409. /**
410. * Find records within a bounding box defined by 4 corners.
411. * Requires numeric indexes on x and y fields.
412. * Corners can be in any order — min/max are extracted automatically.
413. *
414. * @param {Array<{x: number, y: number}>} corners - 4 corner coordinates
415. * @param {function} [filter] - Optional additional filter function
416. * @returns {Cursor}
417. * @example
418. * const results = await db.boundingBox([
419. * { x: -5, y: 5 }, { x: 5, y: 5 },
420. * { x: 5, y: -5 }, { x: -5, y: -5 }
421. * ]);
422. */
423. boundingBox(corners, filter) {
424. const xs = corners.map(c => c.x);
425. const ys = corners.map(c => c.y);
426. const minX = Math.min(...xs), maxX = Math.max(...xs);
427. const minY = Math.min(...ys), maxY = Math.max(...ys);
429. return this.find(filter || null, {
430. index: [
431. { field: 'x', from: minX, to: maxX },
432. { field: 'y', from: minY, to: maxY }
433. ]
434. });
435. }
437. /**
438. * Execute a callback while holding the database lock.
439. * The lock is re-entrant: find/insert/delete/update called inside
440. * the callback reuse the same lock instead of deadlocking.
441. *
442. * Use this for compound operations that must be atomic, e.g.
443. * find-then-insert (login pattern).
444. *
445. * @param {Function} callback - Async function to execute under lock
446. * @returns {Promise<any>} The return value of the callback
447. *
448. * @example
449. * const user = await db.withLock(async () => {
450. * const [existing] = await db.find(u => u.email === email);
451. * if (existing) return existing;
452. * return db.insert({ email });
453. * });
454. */
455. async withLock(callback) {
456. await this.init();
457. await this._acquireLock('withLock');
458. try {
459. return await callback();
460. } finally {
461. await this._releaseLock();
462. }
463. }
465. /**
466. * Generator that yields records from the database file
467. * @param {function|null} [queryFn=null] - Optional filter function
468. * @param {Object} [options] - Generator options
469. * @param {string} [options.mode='record'] - Mode: 'record', 'raw', 'offset', or 'mixed'
470. * @yields {Object|Buffer|Array} Records, buffers, or [record, offset, size] tuples depending on mode
471. */
472. async *recordGenerator(queryFn = null, { mode = 'record', _indexLookup, index } = {}) {
473. await this.init();
474. await this.refresh();
476. // Indexed primary key lookup — O(log n) instead of full scan
477. if (_indexLookup && this._indexManager) {
478. yield* this._indexedLookup(_indexLookup.field, _indexLookup.value, mode);
479. return;
480. }
482. // Index-based find: collect offsets from index(es), then stream only those records
483. if (index && this._indexManager) {
484. yield* this._indexedStream(index, queryFn, mode);
485. return;
486. }
488. // Flush pending writes so reads see all inserted data
489. if (this._dataStream && this._dataStream.writableLength > 0) {
490. await new Promise(resolve => this._dataStream.once('drain', resolve));
491. }
493. // Check if the data file exists before attempting to read it
494. try {
495. await stat(this._dataPath);
496. } catch (error) {
497. // File doesn't exist - return empty generator (no records)
498. return;
499. }
501. // Convert deleted array to Set for O(1) lookup instead of O(n)
502. const deletedSet = new Set(this._meta.deleted);
504. const readStream = createReadStream(this._dataPath);
505. let chunks = [];
506. let totalLength = 0;
507. let processedBytes = 0;
509. for await (const chunk of readStream) {
510. chunks.push(chunk);
511. totalLength += chunk.length;
513. while (true) {
514. // Exit 1: Not enough data to even read the 4-byte size header.
515. if (totalLength < 4) {
516. break;
517. }
519. // Safely read the header, even if it's split across chunks
520. let headerBuffer;
521. if (chunks[0].length >= 4) {
522. headerBuffer = chunks[0];
523. } else {
524. // The header is fragmented, so we must concat just enough to read it.
525. headerBuffer = Buffer.concat(chunks, 4);
526. }
527. const recSize = headerBuffer.readInt32LE(0);
529. // Validate the record size to prevent infinite loops
530. // A record must be at least as large as its header (4 bytes).
531. // A size of 0 or less is invalid and indicates corruption.
532. if (recSize <= 4) {
533. throw new Error(`Invalid record size read from stream: ${recSize}`);
534. }
536. // Exit 2: We have the size, but not the full record yet.
537. if (totalLength < recSize) {
538. break;
539. }
541. // skip deleted records (only after we have the full record)
542. if (deletedSet.has(processedBytes)) {
543. [processedBytes, totalLength] = removeChunk(recSize, processedBytes, totalLength, chunks);
544. continue; // Goes back to while (true)
545. }
547. switch (mode) {
548. case 'raw':
549. yield Buffer.concat(chunks, recSize).subarray(0, recSize);
550. break;
552. case 'mixed':
553. case 'record':
554. const recBuffer = Buffer.concat(chunks, recSize).subarray(0, recSize);
555. // Skip the 4-byte size header
556. const data = deserialize(recBuffer.subarray(4));
557. const rec = this._classToUse
558. ? Object.assign(new this._classToUse(), data)
559. : data;
560. if (queryFn) {
561. if (queryFn(rec)) {
562. yield mode === 'mixed' ? [rec, processedBytes, recSize] : rec;
563. }
564. } else {
565. yield mode === 'mixed' ? [rec, processedBytes, recSize] : rec;
566. }
567. break;
569. case 'offset':
570. // if someone uses a queryFn with offset, we have to deserialize it first to perform the queryFn
571. if (queryFn) {
572. const recBuffer = Buffer.concat(chunks, recSize).subarray(0, recSize);
573. const rec = deserialize(recBuffer.subarray(4));
574. if (queryFn(rec)) {
575. yield [processedBytes, recSize];
576. }
577. } else {
578. yield [processedBytes, recSize];
579. }
580. break;
582. default:
583. throw new Error(`Invalid mode: ${mode}`);
584. }
586. [processedBytes, totalLength] = removeChunk(recSize, processedBytes, totalLength, chunks);
587. }
588. }
589. }
591. /**
592. * Indexed lookup — reads records directly by offset from the index.
593. * Uses binary search on the index file for O(log n) lookups.
594. * @private
595. */
596. async *_indexedLookup(field, value, mode = 'record') {
597. const entries = await this._indexManager.get(field, value);
598. if (entries.length === 0) return;
600. const deletedSet = new Set(this._meta.deleted);
601. const fileHandle = await open(this._dataPath, 'r');
603. try {
604. for (const entry of entries) {
605. const [offset, length] = entry.loc;
606. if (deletedSet.has(offset)) continue;
608. const buffer = Buffer.alloc(length);
609. await fileHandle.read(buffer, 0, length, offset);
611. switch (mode) {
612. case 'raw':
613. yield buffer;
614. break;
615. case 'mixed': {
616. const data = deserialize(buffer.subarray(4));
617. const rec = this._classToUse
618. ? Object.assign(new this._classToUse(), data)
619. : data;
620. yield [rec, offset, length];
621. break;
622. }
623. case 'record':
624. default: {
625. const data = deserialize(buffer.subarray(4));
626. const rec = this._classToUse
627. ? Object.assign(new this._classToUse(), data)
628. : data;
629. yield rec;
630. break;
631. }
632. }
633. }
634. } finally {
635. await fileHandle.close();
636. }
637. }
639. /**
640. * Collect offsets from one or more indexes, optionally intersect, then stream records.
641. * @param {Object|Array} index - Single index hint or array of hints
642. * @param {function|null} queryFn - Optional filter function
643. * @param {string} mode - Output mode: 'record', 'raw', or 'mixed'
644. * @private
645. */
646. async *_indexedStream(index, queryFn, mode = 'record') {
647. const hints = Array.isArray(index) ? index : [index];
648. const deletedSet = new Set(this._meta.deleted);
650. // Collect offset sets from each index hint
651. const offsetSets = [];
652. for (const hint of hints) {
653. const offsets = new Map(); // offset → [offset, length]
654. if (hint.value !== undefined) {
655. // Exact match via get()
656. const entries = await this._indexManager.get(hint.field, hint.value);
657. for (const e of entries) {
658. if (!deletedSet.has(e.loc[0])) offsets.set(e.loc[0], e.loc);
659. }
660. } else {
661. // Range scan via entries()
662. for await (const e of this._indexManager.entries(hint.field, { from: hint.from, to: hint.to, direction: hint.direction })) {
663. if (!deletedSet.has(e.loc[0])) offsets.set(e.loc[0], e.loc);
664. }
665. }
666. offsetSets.push(offsets);
667. }
669. // Intersect: keep only offsets present in ALL sets
670. let locations;
671. if (offsetSets.length === 1) {
672. locations = Array.from(offsetSets[0].values());
673. } else {
674. // Start with smallest set for efficiency
675. offsetSets.sort((a, b) => a.size - b.size);
676. const [smallest, ...rest] = offsetSets;
677. locations = [];
678. for (const [offset, loc] of smallest) {
679. if (rest.every(s => s.has(offset))) locations.push(loc);
680. }
681. }
683. // Stream records from the intersected locations
684. const fileHandle = await open(this._dataPath, 'r');
685. try {
686. for (const [offset, length] of locations) {
687. const buffer = Buffer.alloc(length);
688. await fileHandle.read(buffer, 0, length, offset);
690. if (mode === 'raw') {
691. yield buffer;
692. } else {
693. const data = deserialize(buffer.subarray(4));
694. const rec = this._classToUse
695. ? Object.assign(new this._classToUse(), data)
696. : data;
698. if (queryFn && !queryFn(rec)) continue;
700. if (mode === 'mixed') {
701. yield [rec, offset, length];
702. } else {
703. yield rec;
704. }
705. }
706. }
707. } finally {
708. await fileHandle.close();
709. }
710. }
712. /**
713. * Re-read meta.json from disk so this instance sees changes made by other processes.
714. * Called automatically before every read operation.
715. */
716. async refresh() {
717. try {
718. this._meta = JSON.parse(await readFile(this._metaPath));
719. } catch (e) {
720. // file missing or corrupt — keep current meta
721. }
722. }
724. async persistMeta() {
725. const metaToSave = { ...this._meta };
727. // Only save nextId if we have a numeric primary key
728. if (this._primaryKeyType !== PrimaryKeyType.NUMBER) {
729. delete metaToSave.nextId;
730. }
732. await writeFile(this._metaPath, JSON.stringify(metaToSave));
733. }
735. /**
736. * Compact the database by removing deleted records
737. * This rewrites the data file without tombstones
738. *
739. * @returns {Promise<void>}
740. */
741. async compact() {
742. await this._acquireLock('compact');
743. try {
744. const writeStream = createWriteStream(this._dataPath + '.tmp', { flags: 'w' });
746. for await (const binary of this.find(null, { mode: 'raw' })) {
747. writeStream.write(binary);
748. }
750. await new Promise((resolve, reject) => {
751. writeStream.end((err) => err ? reject(err) : resolve());
752. });
754. // rename is atomic, so we can just rename the file and it will replace the old one
755. await rename(this._dataPath + '.tmp', this._dataPath);
758. this._meta.deleted = [];
759. await this.persistMeta();
761. // Rebuild indexes — offsets changed after compaction
762. if (this._indexManager) {
763. await this._indexManager.init(this._dataPath, { forceRebuild: true });
764. }
765. } finally {
766. await this._releaseLock();
767. }
768. }
770. async _acquireLock(operation, record, retries = 0) {
771. if (this._lockDepth > 0) {
772. this._lockDepth++;
773. return;
774. }
775. try {
776. await writeFile(this._lockPath, String(process.pid), { flag: 'wx' });
777. this._lockDepth = 1;
778. } catch (e) {
779. if (e.code === 'EEXIST') {
780. this.debug(`Waiting for lock file ${operation} retry #${retries}`, record);
781. await new Promise(resolve => setTimeout(resolve, 100));
782. return this._acquireLock(operation, record, retries + 1);
783. }
784. throw e;
785. }
786. }
788. async _releaseLock() {
789. this._lockDepth--;
790. if (this._lockDepth <= 0) {
791. this._lockDepth = 0;
792. await unlink(this._lockPath).catch(() => { });
793. }
794. }
796. /**
797. * Retrieves a document by its file offset and length
798. * @param {Array} location - [offset, length] tuple
799. * @returns {Promise<Object|null>} The deserialized document or null
800. * @private
801. * @deprecated Currently unused - may be removed in future versions
802. */
803. async _getDocByLocation([offset, length]) {
804. if (!offset || length === 0) return null;
805. const fileHandle = await open(this._dataPath, 'r');
806. try {
807. const buffer = Buffer.alloc(length);
808. await fileHandle.read(buffer, 0, length, offset);
809. return deserialize(buffer);
810. } finally {
811. await fileHandle.close();
812. }
813. }
815. /**
816. * Close the database and persist all pending changes
817. * Should be called before process exit
818. *
819. * @returns {Promise<void>}
820. */
821. async close() {
822. // Persist indexes before closing
823. if (this._indexManager) {
824. await this._indexManager.close();
825. }
827. // Close data stream
828. if (this._dataStream) {
829. await new Promise((resolve, reject) => {
830. this._dataStream.end((err) => err ? reject(err) : resolve());
831. });
832. }
834. // Remove signal handlers
835. if (this._processExitHandler) {
836. process.off('SIGINT', this._processExitHandler);
837. process.off('SIGTERM', this._processExitHandler);
838. this._processExitHandler = null;
839. }
840. }
842. debug(...args) {
843. if (this._debug) {
844. console.log(...args);
845. }
846. }
847. }
849. /**
850. * Helper function to remove a processed chunk from the buffer
851. * @param {number} recSize - Size of the record to remove
852. * @param {number} processedBytes - Current offset in the file
853. * @param {number} totalLength - Total length of buffered data
854. * @param {Buffer[]} chunks - Array of buffer chunks
855. * @returns {[number, number]} Updated [processedBytes, totalLength]
856. */
857. function removeChunk(recSize, processedBytes, totalLength, chunks) {
858. processedBytes += recSize;
859. totalLength -= recSize;
861. let bytesToRemove = recSize;
862. while (bytesToRemove > 0 && chunks.length > 0) {
863. const currentChunk = chunks[0];
864. if (bytesToRemove >= currentChunk.length) {
865. bytesToRemove -= currentChunk.length;
866. chunks.shift();
867. } else {
868. chunks[0] = currentChunk.subarray(bytesToRemove);
869. bytesToRemove = 0;
870. }
871. }
873. return [processedBytes, totalLength];
874. }
876. export default MPackDB;