stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

SCRAPE-31: Add scraper indexes and validation collections #30

Merged
stephen merged 2 commits from scraper/indexes into dev 2026-02-07 16:50:51 -07:00
Conversation 1 Commits 2 Files Changed 8 +116 -20
8 changed files with 116 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fe24154518 - Show all commits

24
__tests__/scraper/scraperService.test.js
View File

@ -680,8 +680,16 @@ describe('sanitizeError', () => {
// Test: createScraperIndexes // Test: createScraperIndexes
// ============================================================ // ============================================================
describe('createScraperIndexes', () => { describe('createScraperIndexes', () => {
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
beforeEach(() => {
mockLogger.info.mockClear();
mockLogger.warn.mockClear();
mockLogger.error.mockClear();
});
it('should create the status_startedAt compound index', async () => { it('should create the status_startedAt compound index', async () => {
await createScraperIndexes(db); await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes(); const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt'); const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
@ -690,7 +698,7 @@ describe('createScraperIndexes', () => {
}); });
it('should create the startedAt_desc index', async () => { it('should create the startedAt_desc index', async () => {
await createScraperIndexes(db); await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes(); const indexes = await db.collection('scraper_runs').indexes();
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc'); const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
@ -698,11 +706,17 @@ describe('createScraperIndexes', () => {
expect(startedAtIndex.key).toEqual({ startedAt: -1 }); expect(startedAtIndex.key).toEqual({ startedAt: -1 });
}); });
it('should log success after creating indexes', async () => {
await createScraperIndexes(db, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
});
it('should be idempotent (no error on second call)', async () => { it('should be idempotent (no error on second call)', async () => {
await createScraperIndexes(db); await createScraperIndexes(db, mockLogger);
// Calling again should not throw // Calling again should not throw
await expect(createScraperIndexes(db)).resolves.not.toThrow(); await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
// Indexes should still exist // Indexes should still exist
const indexes = await db.collection('scraper_runs').indexes(); const indexes = await db.collection('scraper_runs').indexes();
@ -720,6 +734,6 @@ describe('createScraperIndexes', () => {
} }
}; };
await expect(createScraperIndexes(faultyDb)).rejects.toThrow('Connection lost'); await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
}); });
}); });

5
services/scraperService.js
View File

@ -740,9 +740,10 @@ async function recordScraperRun(db, runData, logger) {
* Idempotent - safe to call multiple times. * Idempotent - safe to call multiple times.
* *
* @param {Db} db - MongoDB database instance * @param {Db} db - MongoDB database instance
* @param {Object} logger - Logger instance
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
async function createScraperIndexes(db) { async function createScraperIndexes(db, logger) {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS); const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
// Compound index for querying runs by status sorted by most recent // Compound index for querying runs by status sorted by most recent
@ -757,7 +758,7 @@ async function createScraperIndexes(db) {
{ name: 'startedAt_desc' } { name: 'startedAt_desc' }
); );
console.log('Scraper indexes created successfully'); logger.info('Scraper indexes created successfully');
} }
// ============================================================ // ============================================================