Merge dev: Node.js scraper migration + CI fix #32

Merged
stephen merged 32 commits from dev into main 2026-02-08 20:35:29 -07:00
21 changed files with 9281 additions and 25 deletions
Showing only changes of commit c6d480a870 - Show all commits

View File

@ -1404,5 +1404,159 @@ describe('scraperJob - scheduler', () => {
expect(nextRun).not.toBeNull(); expect(nextRun).not.toBeNull();
expect(new Date(nextRun).toISOString()).toBe(nextRun); expect(new Date(nextRun).toISOString()).toBe(nextRun);
}); });
test('should return a time in the future', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
freshInit(mockDb);
const nextRun = freshGetNext();
expect(nextRun).not.toBeNull();
const nextRunDate = new Date(nextRun);
const now = new Date();
expect(nextRunDate.getTime()).toBeGreaterThan(now.getTime());
});
test('should respect configured timezone', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'America/Denver'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
freshInit(mockDb);
const denverNext = freshGetNext();
expect(denverNext).not.toBeNull();
// Get the UTC equivalent for the same schedule
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: utcInit, getNextScheduledRun: utcGetNext } = require('../../jobs/scraperJob');
utcInit(mockDb);
const utcNext = utcGetNext();
expect(utcNext).not.toBeNull();
// Denver is UTC-7 (or UTC-6 during DST), so "6 AM Denver" != "6 AM UTC"
// The two next-run times should be different ISO strings
expect(denverNext).not.toBe(utcNext);
});
test('should return null for invalid cron expression when scheduler was initialized with a modified config', () => {
jest.resetModules();
const mockConfig = {
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
};
jest.doMock('../../config/scraper', () => mockConfig);
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
// Initialize with valid config to set scheduledJob
freshInit(mockDb);
// Now corrupt the schedule config to simulate an invalid expression
// being evaluated at getNextScheduledRun() call time
mockConfig.SCRAPER_SCHEDULE = 'not-a-valid-cron';
const result = freshGetNext();
// cron-parser should throw on invalid expression, handler returns null
expect(result).toBeNull();
});
test('should use cron-parser library to calculate next run', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '30 14 * * 1-5',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
freshInit(mockDb);
const nextRun = freshGetNext();
expect(nextRun).not.toBeNull();
// Verify the time component: the next run should be at 14:30 UTC
const nextDate = new Date(nextRun);
expect(nextDate.getUTCHours()).toBe(14);
expect(nextDate.getUTCMinutes()).toBe(30);
// Verify it falls on a weekday (Mon=1 to Fri=5, getUTCDay: 0=Sun, 1=Mon...5=Fri)
const day = nextDate.getUTCDay();
expect(day).toBeGreaterThanOrEqual(1);
expect(day).toBeLessThanOrEqual(5);
});
}); });
}); });