SCRAPE-13: Create node-cron job initialization (#18)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
@ -1,13 +1,46 @@
|
||||
/**
|
||||
* Tests for scraperJob.js - In-process mutex (isScraperRunning flag)
|
||||
* Tests for scraperJob.js - In-process mutex and scheduler
|
||||
* Following TDD - these tests are written BEFORE implementation
|
||||
*/
|
||||
|
||||
// Mock node-cron before requiring scraperJob
|
||||
jest.mock('node-cron', () => ({
|
||||
validate: jest.fn(),
|
||||
schedule: jest.fn()
|
||||
}));
|
||||
|
||||
// Mock the scraper logger
|
||||
jest.mock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
|
||||
// Mock uuid
|
||||
jest.mock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid-1234')
|
||||
}));
|
||||
|
||||
// Mock scraperService
|
||||
jest.mock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn(() => Promise.resolve({ status: 'success' }))
|
||||
}));
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { createLogger } = require('../../services/scraperLogger');
|
||||
const { runScrape } = require('../../services/scraperService');
|
||||
|
||||
const {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
acquireLock,
|
||||
releaseLock
|
||||
releaseLock,
|
||||
initializeScheduler,
|
||||
stopScheduler,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
describe('scraperJob - mutex lock', () => {
|
||||
@ -147,5 +180,684 @@ describe('scraperJob - mutex lock', () => {
|
||||
test('should export releaseLock as a function', () => {
|
||||
expect(typeof releaseLock).toBe('function');
|
||||
});
|
||||
|
||||
test('should export initializeScheduler as a function', () => {
|
||||
expect(typeof initializeScheduler).toBe('function');
|
||||
});
|
||||
|
||||
test('should export stopScheduler as a function', () => {
|
||||
expect(typeof stopScheduler).toBe('function');
|
||||
});
|
||||
|
||||
test('should export getScheduleExpression as a function', () => {
|
||||
expect(typeof getScheduleExpression).toBe('function');
|
||||
});
|
||||
|
||||
test('should export getNextScheduledRun as a function', () => {
|
||||
expect(typeof getNextScheduledRun).toBe('function');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('scraperJob - scheduler', () => {
|
||||
const mockDb = { collection: jest.fn() };
|
||||
let mockLogger;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset all mocks
|
||||
jest.clearAllMocks();
|
||||
releaseLock();
|
||||
if (typeof stopScheduler === 'function') {
|
||||
stopScheduler();
|
||||
}
|
||||
|
||||
// Setup mock logger
|
||||
mockLogger = {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
createLogger.mockReturnValue(mockLogger);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
releaseLock();
|
||||
if (typeof stopScheduler === 'function') {
|
||||
stopScheduler();
|
||||
}
|
||||
// Reset config module to defaults between tests
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
describe('initializeScheduler()', () => {
|
||||
test('should not initialize when SCRAPER_ENABLED is false', () => {
|
||||
// We need to re-require with modified config
|
||||
jest.resetModules();
|
||||
|
||||
// Mock config with SCRAPER_ENABLED = false
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
// Re-mock the dependencies
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(),
|
||||
schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { createLogger: freshCreateLogger } = require('../../services/scraperLogger');
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
freshCreateLogger.mockReturnValue(freshLogger);
|
||||
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).not.toHaveBeenCalled();
|
||||
expect(freshLogger.info).toHaveBeenCalledWith('Scraper scheduling is disabled');
|
||||
});
|
||||
|
||||
test('should validate cron expression with cron.validate()', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.validate).toHaveBeenCalledWith('0 6 * * *');
|
||||
});
|
||||
|
||||
test('should fall back to default schedule 0 6 * * * if cron expression is invalid', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: 'invalid-cron',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => false),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// After fallback, schedule should be called with default
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 6 * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
expect(freshLogger.error).toHaveBeenCalledWith(
|
||||
'Invalid cron schedule expression',
|
||||
expect.objectContaining({ schedule: 'invalid-cron' })
|
||||
);
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Falling back to default schedule')
|
||||
);
|
||||
});
|
||||
|
||||
test('should enforce minimum 1-hour interval - adjust too-frequent schedules', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '*/5 * * * *', // Every 5 minutes - too frequent
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Should adjust to hourly
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 * * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('less than 1 hour'),
|
||||
expect.objectContaining({ originalSchedule: '*/5 * * * *' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should enforce minimum 1-hour interval for every-minute schedule', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '* * * * *', // Every minute
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 * * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should create cron job with configured timezone', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'America/Denver'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 6 * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'America/Denver' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should log initialization with schedule and next run time', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshLogger.info).toHaveBeenCalledWith(
|
||||
'Scraper scheduler initialized',
|
||||
expect.objectContaining({
|
||||
schedule: '0 6 * * *',
|
||||
timezone: 'UTC'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('should allow valid hourly schedule without adjustment', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 */2 * * *', // Every 2 hours - acceptable
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Should keep the original schedule, not adjust
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 */2 * * *',
|
||||
expect.any(Function),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cron job callback', () => {
|
||||
test('should acquire mutex lock before running scraper', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'scheduled-job-id')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Execute the cron callback
|
||||
await capturedCallback();
|
||||
|
||||
// runScrape should have been called
|
||||
expect(freshRunScrape).toHaveBeenCalledWith(
|
||||
mockDb,
|
||||
expect.objectContaining({ trigger: 'scheduled', jobId: 'scheduled-job-id' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should release mutex lock after completion in finally block', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-release-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
await capturedCallback();
|
||||
|
||||
// Lock should be released after completion
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should release mutex lock even when scraper throws an error', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.reject(new Error('Scrape failed')));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-error-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
await capturedCallback();
|
||||
|
||||
// Lock should be released even after error
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should skip run if scraper is already in progress', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-skip-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Simulate an already-running scraper
|
||||
freshAcquire('existing-job');
|
||||
|
||||
await capturedCallback();
|
||||
|
||||
// runScrape should NOT have been called since lock was already held
|
||||
expect(freshRunScrape).not.toHaveBeenCalled();
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('already in progress')
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stopScheduler()', () => {
|
||||
test('should stop the scheduled cron job', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockStop = jest.fn();
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: mockStop }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
freshStop();
|
||||
|
||||
expect(mockStop).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should be safe to call when no scheduler is active', () => {
|
||||
expect(() => stopScheduler()).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getScheduleExpression()', () => {
|
||||
test('should return "disabled" when SCRAPER_ENABLED is false', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetExpr()).toBe('disabled');
|
||||
});
|
||||
|
||||
test('should return the configured cron expression when enabled', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '30 8 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetExpr()).toBe('30 8 * * *');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNextScheduledRun()', () => {
|
||||
test('should return null when SCRAPER_ENABLED is false', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetNext()).toBeNull();
|
||||
});
|
||||
|
||||
test('should return null when no scheduler is active', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
// No initializeScheduler called, so no scheduledJob
|
||||
expect(freshGetNext()).toBeNull();
|
||||
});
|
||||
|
||||
test('should return an ISO date string when scheduler is active', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
const nextRun = freshGetNext();
|
||||
// Should be a valid ISO date string
|
||||
expect(nextRun).not.toBeNull();
|
||||
expect(new Date(nextRun).toISOString()).toBe(nextRun);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
const cron = require('node-cron');
|
||||
const config = require('../config/scraper');
|
||||
const { runScrape } = require('../services/scraperService');
|
||||
const { createLogger } = require('../services/scraperLogger');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
|
||||
// In-process mutex state
|
||||
let isRunning = false;
|
||||
let currentJobId = null;
|
||||
|
||||
// Scheduler state
|
||||
let scheduledJob = null;
|
||||
|
||||
/**
|
||||
* Check if scraper is currently running
|
||||
* @returns {boolean}
|
||||
@ -40,9 +49,143 @@ function releaseLock() {
|
||||
currentJobId = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured schedule expression
|
||||
* @returns {string} Cron expression or 'disabled'
|
||||
*/
|
||||
function getScheduleExpression() {
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
return 'disabled';
|
||||
}
|
||||
return config.SCRAPER_SCHEDULE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate next scheduled run time
|
||||
* @returns {string|null} ISO timestamp or null if disabled
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
if (!config.SCRAPER_ENABLED || !scheduledJob) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { CronExpressionParser } = require('cron-parser');
|
||||
try {
|
||||
const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
|
||||
tz: config.SCRAPER_TIMEZONE
|
||||
});
|
||||
return interval.next().toISOString();
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if schedule runs more frequently than 1 hour
|
||||
* @param {string} schedule - Cron expression
|
||||
* @returns {boolean} True if schedule is too frequent
|
||||
*/
|
||||
function isScheduleTooFrequent(schedule) {
|
||||
const parts = schedule.trim().split(/\s+/);
|
||||
|
||||
if (parts.length < 5) return false;
|
||||
|
||||
const minuteField = parts[0];
|
||||
|
||||
// If minute field is */N with N < 60, it runs more than once per hour
|
||||
if (/^\*\/\d+$/.test(minuteField)) {
|
||||
const interval = parseInt(minuteField.substring(2), 10);
|
||||
if (interval < 60) return true;
|
||||
}
|
||||
|
||||
// If minute field is *, it runs every minute
|
||||
if (minuteField === '*') return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the scraper scheduler
|
||||
* @param {Db} db - MongoDB database instance
|
||||
*/
|
||||
function initializeScheduler(db) {
|
||||
const logger = createLogger('scheduler');
|
||||
|
||||
// Check if scheduling is enabled
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
logger.info('Scraper scheduling is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(config.SCRAPER_SCHEDULE)) {
|
||||
logger.error('Invalid cron schedule expression', {
|
||||
schedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
logger.warn('Falling back to default schedule: 0 6 * * *');
|
||||
config.SCRAPER_SCHEDULE = '0 6 * * *';
|
||||
}
|
||||
|
||||
// Validate minimum interval (1 hour)
|
||||
if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
|
||||
logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
|
||||
originalSchedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
config.SCRAPER_SCHEDULE = '0 * * * *';
|
||||
}
|
||||
|
||||
// Create the scheduled job
|
||||
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
|
||||
const jobId = uuidv4();
|
||||
const jobLogger = createLogger(jobId);
|
||||
|
||||
jobLogger.info('Scheduled scrape triggered');
|
||||
|
||||
// Check if already running
|
||||
if (!acquireLock(jobId)) {
|
||||
jobLogger.warn('Skipped - scrape already in progress');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await runScrape(db, { trigger: 'scheduled', jobId });
|
||||
} catch (error) {
|
||||
jobLogger.error('Scheduled scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
}, {
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
scheduled: true
|
||||
});
|
||||
|
||||
logger.info('Scraper scheduler initialized', {
|
||||
schedule: config.SCRAPER_SCHEDULE,
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
nextRun: getNextScheduledRun()
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler (for graceful shutdown)
|
||||
*/
|
||||
function stopScheduler() {
|
||||
if (scheduledJob) {
|
||||
scheduledJob.stop();
|
||||
scheduledJob = null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
acquireLock,
|
||||
releaseLock
|
||||
releaseLock,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun,
|
||||
initializeScheduler,
|
||||
stopScheduler
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user