SCRAPE-31: Add scraper indexes and validation collections #30
@ -680,8 +680,16 @@ describe('sanitizeError', () => {
|
||||
// Test: createScraperIndexes
|
||||
// ============================================================
|
||||
describe('createScraperIndexes', () => {
|
||||
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger.info.mockClear();
|
||||
mockLogger.warn.mockClear();
|
||||
mockLogger.error.mockClear();
|
||||
});
|
||||
|
||||
it('should create the status_startedAt compound index', async () => {
|
||||
await createScraperIndexes(db);
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
|
||||
@ -690,7 +698,7 @@ describe('createScraperIndexes', () => {
|
||||
});
|
||||
|
||||
it('should create the startedAt_desc index', async () => {
|
||||
await createScraperIndexes(db);
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
|
||||
@ -698,11 +706,17 @@ describe('createScraperIndexes', () => {
|
||||
expect(startedAtIndex.key).toEqual({ startedAt: -1 });
|
||||
});
|
||||
|
||||
it('should log success after creating indexes', async () => {
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
|
||||
});
|
||||
|
||||
it('should be idempotent (no error on second call)', async () => {
|
||||
await createScraperIndexes(db);
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
// Calling again should not throw
|
||||
await expect(createScraperIndexes(db)).resolves.not.toThrow();
|
||||
await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
|
||||
|
||||
// Indexes should still exist
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
@ -720,6 +734,6 @@ describe('createScraperIndexes', () => {
|
||||
}
|
||||
};
|
||||
|
||||
await expect(createScraperIndexes(faultyDb)).rejects.toThrow('Connection lost');
|
||||
await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
});
|
||||
|
||||
@ -740,9 +740,10 @@ async function recordScraperRun(db, runData, logger) {
|
||||
* Idempotent - safe to call multiple times.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function createScraperIndexes(db) {
|
||||
async function createScraperIndexes(db, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
|
||||
|
||||
// Compound index for querying runs by status sorted by most recent
|
||||
@ -757,7 +758,7 @@ async function createScraperIndexes(db) {
|
||||
{ name: 'startedAt_desc' }
|
||||
);
|
||||
|
||||
console.log('Scraper indexes created successfully');
|
||||
logger.info('Scraper indexes created successfully');
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
|
||||
Reference in New Issue
Block a user