feat: add cron scheduler initialization with validation and mutex integration
All checks were successful
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 38s
All checks were successful
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 38s
Implement initializeScheduler() and stopScheduler() functions for the scraper cron job module. Key features: - Config-driven enable/disable via SCRAPER_ENABLED environment variable - Cron expression validation with fallback to default schedule (0 6 * * *) - Minimum interval enforcement (1-hour floor) to prevent excessive runs - Timezone support via SCRAPER_TIMEZONE configuration - Mutex lock integration to prevent concurrent scraper executions - Automatic lock release in finally block for error resilience - getScheduleExpression() and getNextScheduledRun() status helpers Add comprehensive test suite with 42 tests covering mutex locking, scheduler lifecycle, cron validation, minimum interval enforcement, concurrent execution prevention, and error handling.
This commit is contained in:
@ -1,13 +1,46 @@
|
||||
/**
|
||||
* Tests for scraperJob.js - In-process mutex (isScraperRunning flag)
|
||||
* Tests for scraperJob.js - In-process mutex and scheduler
|
||||
* Following TDD - these tests are written BEFORE implementation
|
||||
*/
|
||||
|
||||
// Mock node-cron before requiring scraperJob
|
||||
jest.mock('node-cron', () => ({
|
||||
validate: jest.fn(),
|
||||
schedule: jest.fn()
|
||||
}));
|
||||
|
||||
// Mock the scraper logger
|
||||
jest.mock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
|
||||
// Mock uuid
|
||||
jest.mock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid-1234')
|
||||
}));
|
||||
|
||||
// Mock scraperService
|
||||
jest.mock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn(() => Promise.resolve({ status: 'success' }))
|
||||
}));
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { createLogger } = require('../../services/scraperLogger');
|
||||
const { runScrape } = require('../../services/scraperService');
|
||||
|
||||
const {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
acquireLock,
|
||||
releaseLock
|
||||
releaseLock,
|
||||
initializeScheduler,
|
||||
stopScheduler,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
describe('scraperJob - mutex lock', () => {
|
||||
@ -147,5 +180,684 @@ describe('scraperJob - mutex lock', () => {
|
||||
test('should export releaseLock as a function', () => {
|
||||
expect(typeof releaseLock).toBe('function');
|
||||
});
|
||||
|
||||
test('should export initializeScheduler as a function', () => {
|
||||
expect(typeof initializeScheduler).toBe('function');
|
||||
});
|
||||
|
||||
test('should export stopScheduler as a function', () => {
|
||||
expect(typeof stopScheduler).toBe('function');
|
||||
});
|
||||
|
||||
test('should export getScheduleExpression as a function', () => {
|
||||
expect(typeof getScheduleExpression).toBe('function');
|
||||
});
|
||||
|
||||
test('should export getNextScheduledRun as a function', () => {
|
||||
expect(typeof getNextScheduledRun).toBe('function');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('scraperJob - scheduler', () => {
|
||||
const mockDb = { collection: jest.fn() };
|
||||
let mockLogger;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset all mocks
|
||||
jest.clearAllMocks();
|
||||
releaseLock();
|
||||
if (typeof stopScheduler === 'function') {
|
||||
stopScheduler();
|
||||
}
|
||||
|
||||
// Setup mock logger
|
||||
mockLogger = {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
createLogger.mockReturnValue(mockLogger);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
releaseLock();
|
||||
if (typeof stopScheduler === 'function') {
|
||||
stopScheduler();
|
||||
}
|
||||
// Reset config module to defaults between tests
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
describe('initializeScheduler()', () => {
|
||||
test('should not initialize when SCRAPER_ENABLED is false', () => {
|
||||
// We need to re-require with modified config
|
||||
jest.resetModules();
|
||||
|
||||
// Mock config with SCRAPER_ENABLED = false
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
// Re-mock the dependencies
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(),
|
||||
schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { createLogger: freshCreateLogger } = require('../../services/scraperLogger');
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
freshCreateLogger.mockReturnValue(freshLogger);
|
||||
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).not.toHaveBeenCalled();
|
||||
expect(freshLogger.info).toHaveBeenCalledWith('Scraper scheduling is disabled');
|
||||
});
|
||||
|
||||
test('should validate cron expression with cron.validate()', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.validate).toHaveBeenCalledWith('0 6 * * *');
|
||||
});
|
||||
|
||||
test('should fall back to default schedule 0 6 * * * if cron expression is invalid', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: 'invalid-cron',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => false),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// After fallback, schedule should be called with default
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 6 * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
expect(freshLogger.error).toHaveBeenCalledWith(
|
||||
'Invalid cron schedule expression',
|
||||
expect.objectContaining({ schedule: 'invalid-cron' })
|
||||
);
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Falling back to default schedule')
|
||||
);
|
||||
});
|
||||
|
||||
test('should enforce minimum 1-hour interval - adjust too-frequent schedules', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '*/5 * * * *', // Every 5 minutes - too frequent
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Should adjust to hourly
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 * * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('less than 1 hour'),
|
||||
expect.objectContaining({ originalSchedule: '*/5 * * * *' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should enforce minimum 1-hour interval for every-minute schedule', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockConfig = {
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '* * * * *', // Every minute
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
};
|
||||
jest.doMock('../../config/scraper', () => mockConfig);
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 * * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'UTC' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should create cron job with configured timezone', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'America/Denver'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 6 * * *',
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ timezone: 'America/Denver' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should log initialization with schedule and next run time', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
expect(freshLogger.info).toHaveBeenCalledWith(
|
||||
'Scraper scheduler initialized',
|
||||
expect.objectContaining({
|
||||
schedule: '0 6 * * *',
|
||||
timezone: 'UTC'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('should allow valid hourly schedule without adjustment', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 */2 * * *', // Every 2 hours - acceptable
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const freshCron = require('node-cron');
|
||||
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Should keep the original schedule, not adjust
|
||||
expect(freshCron.schedule).toHaveBeenCalledWith(
|
||||
'0 */2 * * *',
|
||||
expect.any(Function),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cron job callback', () => {
|
||||
test('should acquire mutex lock before running scraper', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'scheduled-job-id')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Execute the cron callback
|
||||
await capturedCallback();
|
||||
|
||||
// runScrape should have been called
|
||||
expect(freshRunScrape).toHaveBeenCalledWith(
|
||||
mockDb,
|
||||
expect.objectContaining({ trigger: 'scheduled', jobId: 'scheduled-job-id' })
|
||||
);
|
||||
});
|
||||
|
||||
test('should release mutex lock after completion in finally block', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-release-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
await capturedCallback();
|
||||
|
||||
// Lock should be released after completion
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should release mutex lock even when scraper throws an error', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.reject(new Error('Scrape failed')));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-error-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
await capturedCallback();
|
||||
|
||||
// Lock should be released even after error
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should skip run if scraper is already in progress', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'job-skip-test')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Simulate an already-running scraper
|
||||
freshAcquire('existing-job');
|
||||
|
||||
await capturedCallback();
|
||||
|
||||
// runScrape should NOT have been called since lock was already held
|
||||
expect(freshRunScrape).not.toHaveBeenCalled();
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('already in progress')
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stopScheduler()', () => {
|
||||
test('should stop the scheduled cron job', () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockStop = jest.fn();
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: mockStop }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
freshStop();
|
||||
|
||||
expect(mockStop).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should be safe to call when no scheduler is active', () => {
|
||||
expect(() => stopScheduler()).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getScheduleExpression()', () => {
|
||||
test('should return "disabled" when SCRAPER_ENABLED is false', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetExpr()).toBe('disabled');
|
||||
});
|
||||
|
||||
test('should return the configured cron expression when enabled', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '30 8 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetExpr()).toBe('30 8 * * *');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNextScheduledRun()', () => {
|
||||
test('should return null when SCRAPER_ENABLED is false', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshGetNext()).toBeNull();
|
||||
});
|
||||
|
||||
test('should return null when no scheduler is active', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
// No initializeScheduler called, so no scheduledJob
|
||||
expect(freshGetNext()).toBeNull();
|
||||
});
|
||||
|
||||
test('should return an ISO date string when scheduler is active', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC'
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
|
||||
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
|
||||
|
||||
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
const nextRun = freshGetNext();
|
||||
// Should be a valid ISO date string
|
||||
expect(nextRun).not.toBeNull();
|
||||
expect(new Date(nextRun).toISOString()).toBe(nextRun);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
const cron = require('node-cron');
|
||||
const config = require('../config/scraper');
|
||||
const { runScrape } = require('../services/scraperService');
|
||||
const { createLogger } = require('../services/scraperLogger');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
|
||||
// In-process mutex state
|
||||
let isRunning = false;
|
||||
let currentJobId = null;
|
||||
|
||||
// Scheduler state
|
||||
let scheduledJob = null;
|
||||
|
||||
/**
|
||||
* Check if scraper is currently running
|
||||
* @returns {boolean}
|
||||
@ -40,9 +49,143 @@ function releaseLock() {
|
||||
currentJobId = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured schedule expression
|
||||
* @returns {string} Cron expression or 'disabled'
|
||||
*/
|
||||
function getScheduleExpression() {
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
return 'disabled';
|
||||
}
|
||||
return config.SCRAPER_SCHEDULE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate next scheduled run time
|
||||
* @returns {string|null} ISO timestamp or null if disabled
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
if (!config.SCRAPER_ENABLED || !scheduledJob) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { CronExpressionParser } = require('cron-parser');
|
||||
try {
|
||||
const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
|
||||
tz: config.SCRAPER_TIMEZONE
|
||||
});
|
||||
return interval.next().toISOString();
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if schedule runs more frequently than 1 hour
|
||||
* @param {string} schedule - Cron expression
|
||||
* @returns {boolean} True if schedule is too frequent
|
||||
*/
|
||||
function isScheduleTooFrequent(schedule) {
|
||||
const parts = schedule.trim().split(/\s+/);
|
||||
|
||||
if (parts.length < 5) return false;
|
||||
|
||||
const minuteField = parts[0];
|
||||
|
||||
// If minute field is */N with N < 60, it runs more than once per hour
|
||||
if (/^\*\/\d+$/.test(minuteField)) {
|
||||
const interval = parseInt(minuteField.substring(2), 10);
|
||||
if (interval < 60) return true;
|
||||
}
|
||||
|
||||
// If minute field is *, it runs every minute
|
||||
if (minuteField === '*') return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the scraper scheduler
|
||||
* @param {Db} db - MongoDB database instance
|
||||
*/
|
||||
function initializeScheduler(db) {
|
||||
const logger = createLogger('scheduler');
|
||||
|
||||
// Check if scheduling is enabled
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
logger.info('Scraper scheduling is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(config.SCRAPER_SCHEDULE)) {
|
||||
logger.error('Invalid cron schedule expression', {
|
||||
schedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
logger.warn('Falling back to default schedule: 0 6 * * *');
|
||||
config.SCRAPER_SCHEDULE = '0 6 * * *';
|
||||
}
|
||||
|
||||
// Validate minimum interval (1 hour)
|
||||
if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
|
||||
logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
|
||||
originalSchedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
config.SCRAPER_SCHEDULE = '0 * * * *';
|
||||
}
|
||||
|
||||
// Create the scheduled job
|
||||
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
|
||||
const jobId = uuidv4();
|
||||
const jobLogger = createLogger(jobId);
|
||||
|
||||
jobLogger.info('Scheduled scrape triggered');
|
||||
|
||||
// Check if already running
|
||||
if (!acquireLock(jobId)) {
|
||||
jobLogger.warn('Skipped - scrape already in progress');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await runScrape(db, { trigger: 'scheduled', jobId });
|
||||
} catch (error) {
|
||||
jobLogger.error('Scheduled scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
}, {
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
scheduled: true
|
||||
});
|
||||
|
||||
logger.info('Scraper scheduler initialized', {
|
||||
schedule: config.SCRAPER_SCHEDULE,
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
nextRun: getNextScheduledRun()
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler (for graceful shutdown)
|
||||
*/
|
||||
function stopScheduler() {
|
||||
if (scheduledJob) {
|
||||
scheduledJob.stop();
|
||||
scheduledJob = null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
acquireLock,
|
||||
releaseLock
|
||||
releaseLock,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun,
|
||||
initializeScheduler,
|
||||
stopScheduler
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user