mpackdb

5.8 KB

1. import MPackDB from '../src/MPackDB.js';
2. import { rm } from 'fs/promises';
3. import { spawn } from 'child_process';
4. import { fileURLToPath } from 'url';
5. import { dirname, join } from 'path';
7. const TEST_DB_PATH = 'db/multi-process';
8. const WORKER = join(dirname(fileURLToPath(import.meta.url)), 'helpers/multi-process-worker.js');
10. console.log('--- Multi-Process Write Test ---');
11. console.log('Several separate node processes write to the same database files.');
12. console.log('This is the mpackdb-admin scenario: a second process must be able');
13. console.log('to write while others read/write the same db.\n');
15. await rm(TEST_DB_PATH, { recursive: true, force: true });
17. function runWorker(dbPath, workerId, mode) {
18. return new Promise((resolve, reject) => {
19. const child = spawn(process.execPath, [WORKER, dbPath, String(workerId), mode], { stdio: ['ignore', 'pipe', 'inherit'] });
20. let out = '';
21. child.stdout.on('data', d => out += d);
22. child.on('exit', code => code === 0 ? resolve(out) : reject(new Error(`worker ${workerId} (${mode}) exited with ${code}`)));
23. });
24. }
26. const sleep = ms => new Promise(r => setTimeout(r, ms));
27. let failures = 0;
29. // ============================================================
30. // PHASE 1: 3 concurrent processes insert + delete on a string-PK store
31. // ============================================================
32. console.log('PHASE 1: 3 concurrent worker processes, 15 inserts + 5 deletes each');
33. const utxoPath = `${TEST_DB_PATH}/utxos`;
34. await Promise.all([1, 2, 3].map(w => runWorker(utxoPath, w, 'utxo')));
36. const utxos = new MPackDB(utxoPath, { primaryKey: 'id', indexes: ['symbol', '*blockNumber'], compact: false });
38. for (const w of [1, 2, 3]) {
39. for (let i = 0; i < 15; i++) {
40. const deleted = i % 3 === 0;
41. const found = await utxos.find(`w${w}-${i}`);
42. const expected = deleted ? 0 : 1;
43. if (found.length !== expected) {
44. failures++;
45. console.log(` ✗ w${w}-${i}: indexed find returned ${found.length}, expected ${expected}`);
46. }
47. }
48. // Secondary index must see the other processes' records too
49. const bySymbol = await utxos.find(null, { index: { field: 'symbol', value: `SYM${w}` } });
50. if (bySymbol.length !== 10) {
51. failures++;
52. console.log(` ✗ symbol index for SYM${w}: ${bySymbol.length} records, expected 10`);
53. }
54. }
55. const fullScan = await utxos.find(() => true);
56. console.log(` full scan: ${fullScan.length} records (expected: 30)`);
57. if (fullScan.length !== 30) failures++;
58. console.log(` ${failures === 0 ? '✓ PASS' : '✗ FAIL'}\n`);
59. await utxos.close();
61. // ============================================================
62. // PHASE 2: numeric auto-increment PK across processes — no id collisions
63. // ============================================================
64. console.log('PHASE 2: 3 concurrent worker processes on a numeric-PK store');
65. const numPath = `${TEST_DB_PATH}/numeric`;
66. const outputs = await Promise.all([1, 2, 3].map(w => runWorker(numPath, w, 'numeric')));
67. const allIds = outputs.flatMap(o => JSON.parse(o.trim()));
68. const uniqueIds = new Set(allIds);
69. console.log(` ${allIds.length} inserts, ${uniqueIds.size} unique ids (expected: 45 / 45)`);
70. const phase2Fail = allIds.length !== 45 || uniqueIds.size !== 45;
71. if (phase2Fail) failures++;
73. const numDb = new MPackDB(numPath, { primaryKey: '*id', compact: false });
74. const numRecords = await numDb.find();
75. console.log(` records on disk: ${numRecords.length} (expected: 45)`);
76. if (numRecords.length !== 45) failures++;
77. console.log(` ${!phase2Fail && numRecords.length === 45 ? '✓ PASS' : '✗ FAIL'}\n`);
78. await numDb.close();
80. // ============================================================
81. // PHASE 3: live visibility — a worker inserts (never persisting its indexes,
82. // then dying without close) while the parent's already-open instance watches
83. // ============================================================
84. console.log('PHASE 3: live cross-process visibility without index persistence');
85. const livePath = `${TEST_DB_PATH}/live`;
86. const liveDb = new MPackDB(livePath, { primaryKey: 'id', indexes: ['symbol'], compact: false });
87. await liveDb.init();
89. const liveWorker = spawn(process.execPath, [WORKER, livePath, '9', 'live'], { stdio: ['pipe', 'pipe', 'inherit'] });
90. await new Promise((resolve, reject) => {
91. liveWorker.stdout.on('data', d => { if (String(d).includes('INSERTED')) resolve(); });
92. liveWorker.on('exit', () => reject(new Error('live worker died before inserting')));
93. setTimeout(() => reject(new Error('live worker timeout')), 15000);
94. });
96. // Worker inserted 5 records but never persisted its indexes. Our instance must
97. // pick them up via meta/data refresh + index tail scan.
98. let liveSeen = 0;
99. for (let attempt = 0; attempt < 40; attempt++) {
100. const seen = await liveDb.find(null, { index: { field: 'symbol', value: 'LIVE' } });
101. liveSeen = seen.length;
102. if (liveSeen === 5) break;
103. await sleep(250);
104. }
105. console.log(` parent sees ${liveSeen}/5 live records via secondary index`);
106. if (liveSeen !== 5) failures++;
108. const livePk = await liveDb.find('live-3');
109. console.log(` primary key lookup live-3: ${livePk.length} (expected: 1)`);
110. if (livePk.length !== 1) failures++;
112. liveWorker.stdin.write('exit\n');
113. await new Promise(resolve => liveWorker.on('exit', resolve));
115. // Worker died without close() — a FRESH instance must still index everything
116. const freshDb = new MPackDB(livePath, { primaryKey: 'id', indexes: ['symbol'], compact: false });
117. const freshSeen = await freshDb.find(null, { index: { field: 'symbol', value: 'LIVE' } });
118. console.log(` fresh instance after worker crash: ${freshSeen.length}/5 via index`);
119. if (freshSeen.length !== 5) failures++;
120. console.log(` ${liveSeen === 5 && livePk.length === 1 && freshSeen.length === 5 ? '✓ PASS' : '✗ FAIL'}\n`);
121. await freshDb.close();
122. await liveDb.close();
124. // ============================================================
125. console.log(failures === 0 ? '✓ All multi-process write tests passed!' : `✗ ${failures} multi-process checks FAILED`);
126. if (failures > 0) throw new Error('multi-process-write tests FAILED');