SCRAPE-10: Implement recordScraperRun() for scraper_runs (#15)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
307
__tests__/scraper/recordScraperRun.test.js
Normal file
307
__tests__/scraper/recordScraperRun.test.js
Normal file
@ -0,0 +1,307 @@
|
||||
/**
|
||||
* Tests for recordScraperRun() history persistence
|
||||
*
|
||||
* Covers:
|
||||
* - insertOne is called with runData plus recordedAt timestamp
|
||||
* - All runData fields are preserved (jobId, trigger, status, duration, etc.)
|
||||
* - Error is caught and logged but NOT thrown (graceful failure)
|
||||
* - Returns null on error instead of crashing
|
||||
* - Returns insert result on success
|
||||
* - recordedAt is a valid Date object
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require after implementation
|
||||
let recordScraperRun;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
let logger;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ recordScraperRun } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean all collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
|
||||
// Create fresh mock logger for each test
|
||||
logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
});
|
||||
|
||||
describe('recordScraperRun', () => {
|
||||
const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
|
||||
|
||||
// Sample runData matching the structure from runScrape()'s finally block
|
||||
function createSampleRunData(overrides = {}) {
|
||||
return {
|
||||
jobId: 'test-job-001',
|
||||
trigger: 'manual',
|
||||
dryRun: false,
|
||||
status: 'success',
|
||||
startedAt: '2026-02-06T06:00:00.000Z',
|
||||
completedAt: '2026-02-06T06:00:12.345Z',
|
||||
duration: 12345,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 48,
|
||||
newUnitsCount: 3,
|
||||
rentedUnitsCount: 1,
|
||||
staleUnitsCount: 2,
|
||||
errors: [],
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. insertOne is called with runData plus recordedAt timestamp
|
||||
// ---------------------------------------------------------------
|
||||
describe('insertOne with runData and recordedAt', () => {
|
||||
it('should insert a document into the scraper_runs collection', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should include a recordedAt field in the inserted document', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.recordedAt).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. All runData fields are preserved
|
||||
// ---------------------------------------------------------------
|
||||
describe('preserving all runData fields', () => {
|
||||
it('should preserve jobId, trigger, status, and duration', async () => {
|
||||
const runData = createSampleRunData({
|
||||
jobId: 'preserve-test-001',
|
||||
trigger: 'scheduled',
|
||||
status: 'failed',
|
||||
duration: 9999
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.jobId).toBe('preserve-test-001');
|
||||
expect(doc.trigger).toBe('scheduled');
|
||||
expect(doc.status).toBe('failed');
|
||||
expect(doc.duration).toBe(9999);
|
||||
});
|
||||
|
||||
it('should preserve unit processing metrics', async () => {
|
||||
const runData = createSampleRunData({
|
||||
unitsProcessed: 42,
|
||||
pricesInserted: 40,
|
||||
newUnitsCount: 5,
|
||||
rentedUnitsCount: 2,
|
||||
staleUnitsCount: 3
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.unitsProcessed).toBe(42);
|
||||
expect(doc.pricesInserted).toBe(40);
|
||||
expect(doc.newUnitsCount).toBe(5);
|
||||
expect(doc.rentedUnitsCount).toBe(2);
|
||||
expect(doc.staleUnitsCount).toBe(3);
|
||||
});
|
||||
|
||||
it('should preserve timing fields (startedAt, completedAt)', async () => {
|
||||
const runData = createSampleRunData({
|
||||
startedAt: '2026-02-06T10:00:00.000Z',
|
||||
completedAt: '2026-02-06T10:00:15.500Z'
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
|
||||
expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
|
||||
});
|
||||
|
||||
it('should preserve dryRun flag', async () => {
|
||||
const runData = createSampleRunData({ dryRun: true });
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.dryRun).toBe(true);
|
||||
});
|
||||
|
||||
it('should preserve errors array when it has entries', async () => {
|
||||
const runData = createSampleRunData({
|
||||
status: 'failed',
|
||||
errors: ['Connection timeout', 'Retry exhausted']
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
|
||||
});
|
||||
|
||||
it('should preserve empty errors array for successful runs', async () => {
|
||||
const runData = createSampleRunData({ errors: [] });
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Error is caught and logged but NOT thrown (graceful failure)
|
||||
// ---------------------------------------------------------------
|
||||
describe('graceful error handling', () => {
|
||||
it('should NOT throw when insertOne fails', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
// Create a mock db that throws on insertOne
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
// This should NOT throw
|
||||
await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should log error message when insert fails', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await recordScraperRun(mockDb, runData, logger);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to record scraper run',
|
||||
{ errorMessage: 'Disk full' }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Returns null on error instead of crashing
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value on error', () => {
|
||||
it('should return null when insertOne throws', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const result = await recordScraperRun(mockDb, runData, logger);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Returns insert result on success
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value on success', () => {
|
||||
it('should return the insertOne result object', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('should return result with acknowledged property', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result.acknowledged).toBe(true);
|
||||
});
|
||||
|
||||
it('should return result with insertedId', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result.insertedId).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. recordedAt is a valid Date object
|
||||
// ---------------------------------------------------------------
|
||||
describe('recordedAt field', () => {
|
||||
it('should set recordedAt as a Date instance', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.recordedAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it('should set recordedAt close to current time', async () => {
|
||||
const beforeTime = new Date();
|
||||
|
||||
const runData = createSampleRunData();
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const afterTime = new Date();
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
|
||||
expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
|
||||
});
|
||||
|
||||
it('should not overwrite any existing runData fields with recordedAt', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
|
||||
// Verify the original fields still exist alongside recordedAt
|
||||
expect(doc.jobId).toBe(runData.jobId);
|
||||
expect(doc.status).toBe(runData.status);
|
||||
expect(doc.recordedAt).toBeInstanceOf(Date);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user