mpackdb

7.9 KB

1. import MPackDB from '../src/MPackDB.js';
2. import { rm } from 'fs/promises';
3. import { statSync, readFileSync } from 'fs';
5. const TEST_DB_PATH = 'db/compaction';
7. console.log('='.repeat(60));
8. console.log('Compaction Test - Large Dataset');
9. console.log('='.repeat(60));
11. // Clean up
12. await rm(TEST_DB_PATH, { recursive: true, force: true });
14. // Create DB with indexes
15. const db = new MPackDB(`${TEST_DB_PATH}/records`, {
16. primaryKey: '*id',
17. indexes: ['*id', 'email', '*score'],
18. indexPersistThreshold: 100, // Low threshold - persist indexes
19. indexPersistInterval: 0 // Disable periodic persist
20. });
22. console.log('\n1. Insert 10,000 records');
23. const startInsert = Date.now();
24. for (let i = 0; i < 10_000; i++) {
25. await db.insert({
26. email: `user${i}@example.com`,
27. name: `User ${i}`,
28. score: Math.floor(Math.random() * 100),
29. data: `Some data for record ${i}`.repeat(5) // Add some bulk
30. });
31. }
32. const insertTime = Date.now() - startInsert;
33. console.log(`✓ Inserted 10,000 records in ${insertTime}ms (${(insertTime / 10_000).toFixed(2)}ms per record)`);
35. // Get initial file sizes
36. const initialDataSize = statSync(`${TEST_DB_PATH}/records.mpack`).size;
37. const initialIdIndexSize = statSync(`${TEST_DB_PATH}/records.id.txt`).size;
38. const initialEmailIndexSize = statSync(`${TEST_DB_PATH}/records.email.txt`).size;
39. const initialScoreIndexSize = statSync(`${TEST_DB_PATH}/records.score.txt`).size;
41. console.log(`\n2. Initial file sizes:`);
42. console.log(` Data file: ${(initialDataSize / 1024).toFixed(2)} KB`);
43. console.log(` ID index: ${(initialIdIndexSize / 1024).toFixed(2)} KB`);
44. console.log(` Email index: ${(initialEmailIndexSize / 1024).toFixed(2)} KB`);
45. console.log(` Score index: ${(initialScoreIndexSize / 1024).toFixed(2)} KB`);
47. // Randomly select 500 IDs to delete
48. console.log('\n3. Delete 500 random records');
49. const idsToDelete = new Set();
50. while (idsToDelete.size < 500) {
51. idsToDelete.add(Math.floor(Math.random() * 10_000));
52. }
54. const startDelete = Date.now();
55. let actualDeleted = 0;
56. for (const id of idsToDelete) {
57. const deletedRecords = await db.delete(r => r.id === id);
58. actualDeleted += deletedRecords.length;
59. }
60. const deleteTime = Date.now() - startDelete;
61. console.log(`✓ Deleted ${actualDeleted} records in ${deleteTime}ms`);
63. // Check meta.deleted array
64. const metaBeforeCompact = JSON.parse(readFileSync(`${TEST_DB_PATH}/records.meta.json`, 'utf-8'));
65. console.log(`✓ Meta.deleted has ${metaBeforeCompact.deleted.length} entries`);
67. // File sizes before compaction (should be same as initial)
68. const beforeCompactDataSize = statSync(`${TEST_DB_PATH}/records.mpack`).size;
69. console.log(`✓ Data file still ${(beforeCompactDataSize / 1024).toFixed(2)} KB (tombstones not removed yet)`);
71. // Verify we have the correct number of records
72. const expectedRemaining = 10_000 - actualDeleted;
73. let countBeforeCompact = 0;
74. for await (const record of db.find()) {
75. countBeforeCompact++;
76. }
77. console.log(`✓ Query returns ${countBeforeCompact} records (should be ${expectedRemaining})`);
78. if (countBeforeCompact !== expectedRemaining) {
79. throw new Error(`Expected ${expectedRemaining} records, got ${countBeforeCompact}`);
80. }
82. // Run compaction
83. console.log('\n4. Run compaction');
84. const startCompact = Date.now();
85. await db.compact();
86. const compactTime = Date.now() - startCompact;
87. console.log(`✓ Compaction completed in ${compactTime}ms`);
89. // Check file sizes after compaction
90. const afterCompactDataSize = statSync(`${TEST_DB_PATH}/records.mpack`).size;
91. const afterCompactIdIndexSize = statSync(`${TEST_DB_PATH}/records.id.txt`).size;
92. const afterCompactEmailIndexSize = statSync(`${TEST_DB_PATH}/records.email.txt`).size;
93. const afterCompactScoreIndexSize = statSync(`${TEST_DB_PATH}/records.score.txt`).size;
95. console.log(`\n5. File sizes after compaction:`);
96. console.log(` Data file: ${(afterCompactDataSize / 1024).toFixed(2)} KB (was ${(beforeCompactDataSize / 1024).toFixed(2)} KB)`);
97. console.log(` Reduction: ${(((beforeCompactDataSize - afterCompactDataSize) / beforeCompactDataSize) * 100).toFixed(1)}%`);
98. console.log(` ID index: ${(afterCompactIdIndexSize / 1024).toFixed(2)} KB`);
99. console.log(` Email index: ${(afterCompactEmailIndexSize / 1024).toFixed(2)} KB`);
100. console.log(` Score index: ${(afterCompactScoreIndexSize / 1024).toFixed(2)} KB`);
102. // Verify data file is smaller
103. if (afterCompactDataSize >= beforeCompactDataSize) {
104. throw new Error(`Data file should be smaller after compaction! Before: ${beforeCompactDataSize}, After: ${afterCompactDataSize}`);
105. }
106. console.log(`✓ Data file reduced by ${((beforeCompactDataSize - afterCompactDataSize) / 1024).toFixed(2)} KB`);
108. // Check meta.deleted is empty
109. const metaAfterCompact = JSON.parse(readFileSync(`${TEST_DB_PATH}/records.meta.json`, 'utf-8'));
110. console.log(`\n6. Verify meta.json:`);
111. console.log(` deleted array length: ${metaAfterCompact.deleted.length} (should be 0)`);
112. console.log(` nextId: ${metaAfterCompact.nextId} (should be 10,000)`);
114. if (metaAfterCompact.deleted.length !== 0) {
115. throw new Error(`Meta.deleted should be empty after compaction, got ${metaAfterCompact.deleted.length} entries`);
116. }
117. if (metaAfterCompact.nextId !== 10_000) {
118. throw new Error(`Meta.nextId should be 10,000, got ${metaAfterCompact.nextId}`);
119. }
120. console.log(`✓ Meta.deleted is empty`);
121. console.log(`✓ Meta.nextId is correct`);
123. // Verify we still have the correct number of records
124. let countAfterCompact = 0;
125. for await (const record of db.find()) {
126. countAfterCompact++;
127. }
128. console.log(`\n7. Verify data integrity:`);
129. console.log(` Record count: ${countAfterCompact} (should be ${expectedRemaining})`);
130. if (countAfterCompact !== expectedRemaining) {
131. throw new Error(`Expected ${expectedRemaining} records after compaction, got ${countAfterCompact}`);
132. }
133. console.log(`✓ All ${expectedRemaining} records intact`);
135. // Verify deleted records are gone
136. console.log(`\n8. Verify deleted records are gone:`);
137. let foundDeletedRecord = false;
138. for (const id of idsToDelete) {
139. const result = await db.find(id);
140. if (result.length > 0) {
141. console.log(`❌ Found deleted record with id ${id}`);
142. foundDeletedRecord = true;
143. break;
144. }
145. }
146. if (foundDeletedRecord) {
147. throw new Error('Found a record that should have been deleted!');
148. }
149. console.log(`✓ All ${actualDeleted} deleted records are gone`);
151. // Verify indexes are consistent
152. console.log(`\n9. Verify indexes:`);
153. const uniqueIds = new Set();
154. const uniqueEmails = new Set();
155. for await (const record of db.find()) {
156. uniqueIds.add(record.id);
157. uniqueEmails.add(record.email);
158. }
160. console.log(` Unique IDs: ${uniqueIds.size} (should be ${expectedRemaining})`);
161. console.log(` Unique emails: ${uniqueEmails.size} (should be ${expectedRemaining})`);
163. if (uniqueIds.size !== expectedRemaining) {
164. throw new Error(`Expected ${expectedRemaining} unique IDs, got ${uniqueIds.size}`);
165. }
166. if (uniqueEmails.size !== expectedRemaining) {
167. throw new Error(`Expected ${expectedRemaining} unique emails, got ${uniqueEmails.size}`);
168. }
169. console.log(`✓ Indexes are consistent`);
171. // Test query performance after compaction
172. console.log(`\n10. Test query performance:`);
173. const testId = Array.from(uniqueIds)[Math.floor(Math.random() * uniqueIds.size)];
174. const startQuery = Date.now();
175. const queryResult = await db.find(testId);
176. const queryTime = Date.now() - startQuery;
177. console.log(` Find by ID (${testId}): ${queryTime}ms`);
178. console.log(` Result: ${queryResult[0]?.name}`);
179. if (queryResult.length !== 1) {
180. throw new Error(`Expected 1 result, got ${queryResult.length}`);
181. }
182. console.log(`✓ Queries work correctly after compaction`);
184. await db.close();
186. console.log('\n' + '='.repeat(60));
187. console.log('✓ Compaction test passed!');
188. console.log('='.repeat(60));
189. console.log(`\nSummary:`);
190. console.log(` - Inserted 10,000 records`);
191. console.log(` - Deleted ${actualDeleted} records (${((actualDeleted / 10_000) * 100).toFixed(1)}%)`);
192. console.log(` - Compaction reduced file by ${(((beforeCompactDataSize - afterCompactDataSize) / beforeCompactDataSize) * 100).toFixed(1)}%`);
193. console.log(` - All data integrity checks passed`);
194. console.log(` - Indexes remain consistent`);