feat: add cron scheduler initialization with validation and mutex integration
All checks were successful
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 38s

Implement initializeScheduler() and stopScheduler() functions for the
scraper cron job module. Key features:

- Config-driven enable/disable via SCRAPER_ENABLED environment variable
- Cron expression validation with fallback to default schedule (0 6 * * *)
- Minimum interval enforcement (1-hour floor) to prevent excessive runs
- Timezone support via SCRAPER_TIMEZONE configuration
- Mutex lock integration to prevent concurrent scraper executions
- Automatic lock release in finally block for error resilience
- getScheduleExpression() and getNextScheduledRun() status helpers

Add comprehensive test suite with 42 tests covering mutex locking,
scheduler lifecycle, cron validation, minimum interval enforcement,
concurrent execution prevention, and error handling.
This commit is contained in:
2026-02-06 16:34:49 -07:00
parent dd0269eb30
commit b0163d94fb
2 changed files with 858 additions and 3 deletions

View File

@ -1,13 +1,46 @@
/**
* Tests for scraperJob.js - In-process mutex (isScraperRunning flag)
* Tests for scraperJob.js - In-process mutex and scheduler
* Following TDD - these tests are written BEFORE implementation
*/
// Mock node-cron before requiring scraperJob
jest.mock('node-cron', () => ({
validate: jest.fn(),
schedule: jest.fn()
}));
// Mock the scraper logger
jest.mock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
}))
}));
// Mock uuid
jest.mock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid-1234')
}));
// Mock scraperService
jest.mock('../../services/scraperService', () => ({
runScrape: jest.fn(() => Promise.resolve({ status: 'success' }))
}));
const cron = require('node-cron');
const { createLogger } = require('../../services/scraperLogger');
const { runScrape } = require('../../services/scraperService');
const {
isScraperRunning,
getCurrentJobId,
acquireLock,
releaseLock
releaseLock,
initializeScheduler,
stopScheduler,
getScheduleExpression,
getNextScheduledRun
} = require('../../jobs/scraperJob');
describe('scraperJob - mutex lock', () => {
@ -147,5 +180,684 @@ describe('scraperJob - mutex lock', () => {
test('should export releaseLock as a function', () => {
expect(typeof releaseLock).toBe('function');
});
test('should export initializeScheduler as a function', () => {
expect(typeof initializeScheduler).toBe('function');
});
test('should export stopScheduler as a function', () => {
expect(typeof stopScheduler).toBe('function');
});
test('should export getScheduleExpression as a function', () => {
expect(typeof getScheduleExpression).toBe('function');
});
test('should export getNextScheduledRun as a function', () => {
expect(typeof getNextScheduledRun).toBe('function');
});
});
});
describe('scraperJob - scheduler', () => {
const mockDb = { collection: jest.fn() };
let mockLogger;
beforeEach(() => {
// Reset all mocks
jest.clearAllMocks();
releaseLock();
if (typeof stopScheduler === 'function') {
stopScheduler();
}
// Setup mock logger
mockLogger = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
createLogger.mockReturnValue(mockLogger);
});
afterEach(() => {
releaseLock();
if (typeof stopScheduler === 'function') {
stopScheduler();
}
// Reset config module to defaults between tests
jest.resetModules();
});
describe('initializeScheduler()', () => {
test('should not initialize when SCRAPER_ENABLED is false', () => {
// We need to re-require with modified config
jest.resetModules();
// Mock config with SCRAPER_ENABLED = false
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: false,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
// Re-mock the dependencies
jest.doMock('node-cron', () => ({
validate: jest.fn(),
schedule: jest.fn()
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
}))
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { createLogger: freshCreateLogger } = require('../../services/scraperLogger');
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
freshCreateLogger.mockReturnValue(freshLogger);
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
expect(freshCron.schedule).not.toHaveBeenCalled();
expect(freshLogger.info).toHaveBeenCalledWith('Scraper scheduling is disabled');
});
test('should validate cron expression with cron.validate()', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
expect(freshCron.validate).toHaveBeenCalledWith('0 6 * * *');
});
test('should fall back to default schedule 0 6 * * * if cron expression is invalid', () => {
jest.resetModules();
const mockConfig = {
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: 'invalid-cron',
SCRAPER_TIMEZONE: 'UTC'
};
jest.doMock('../../config/scraper', () => mockConfig);
jest.doMock('node-cron', () => ({
validate: jest.fn(() => false),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
// After fallback, schedule should be called with default
expect(freshCron.schedule).toHaveBeenCalledWith(
'0 6 * * *',
expect.any(Function),
expect.objectContaining({ timezone: 'UTC' })
);
expect(freshLogger.error).toHaveBeenCalledWith(
'Invalid cron schedule expression',
expect.objectContaining({ schedule: 'invalid-cron' })
);
expect(freshLogger.warn).toHaveBeenCalledWith(
expect.stringContaining('Falling back to default schedule')
);
});
test('should enforce minimum 1-hour interval - adjust too-frequent schedules', () => {
jest.resetModules();
const mockConfig = {
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '*/5 * * * *', // Every 5 minutes - too frequent
SCRAPER_TIMEZONE: 'UTC'
};
jest.doMock('../../config/scraper', () => mockConfig);
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
// Should adjust to hourly
expect(freshCron.schedule).toHaveBeenCalledWith(
'0 * * * *',
expect.any(Function),
expect.objectContaining({ timezone: 'UTC' })
);
expect(freshLogger.warn).toHaveBeenCalledWith(
expect.stringContaining('less than 1 hour'),
expect.objectContaining({ originalSchedule: '*/5 * * * *' })
);
});
test('should enforce minimum 1-hour interval for every-minute schedule', () => {
jest.resetModules();
const mockConfig = {
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '* * * * *', // Every minute
SCRAPER_TIMEZONE: 'UTC'
};
jest.doMock('../../config/scraper', () => mockConfig);
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
expect(freshCron.schedule).toHaveBeenCalledWith(
'0 * * * *',
expect.any(Function),
expect.objectContaining({ timezone: 'UTC' })
);
});
test('should create cron job with configured timezone', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'America/Denver'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
expect(freshCron.schedule).toHaveBeenCalledWith(
'0 6 * * *',
expect.any(Function),
expect.objectContaining({ timezone: 'America/Denver' })
);
});
test('should log initialization with schedule and next run time', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
expect(freshLogger.info).toHaveBeenCalledWith(
'Scraper scheduler initialized',
expect.objectContaining({
schedule: '0 6 * * *',
timezone: 'UTC'
})
);
});
test('should allow valid hourly schedule without adjustment', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 */2 * * *', // Every 2 hours - acceptable
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
freshInit(mockDb);
// Should keep the original schedule, not adjust
expect(freshCron.schedule).toHaveBeenCalledWith(
'0 */2 * * *',
expect.any(Function),
expect.any(Object)
);
});
});
describe('cron job callback', () => {
test('should acquire mutex lock before running scraper', async () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
let capturedCallback;
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn((expr, cb) => {
capturedCallback = cb;
return { stop: jest.fn() };
})
}));
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
jest.doMock('../../services/scraperService', () => ({
runScrape: freshRunScrape
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'scheduled-job-id')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
freshInit(mockDb);
// Execute the cron callback
await capturedCallback();
// runScrape should have been called
expect(freshRunScrape).toHaveBeenCalledWith(
mockDb,
expect.objectContaining({ trigger: 'scheduled', jobId: 'scheduled-job-id' })
);
});
test('should release mutex lock after completion in finally block', async () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
let capturedCallback;
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn((expr, cb) => {
capturedCallback = cb;
return { stop: jest.fn() };
})
}));
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
jest.doMock('../../services/scraperService', () => ({
runScrape: freshRunScrape
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-release-test')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
freshInit(mockDb);
await capturedCallback();
// Lock should be released after completion
expect(freshIsRunning()).toBe(false);
});
test('should release mutex lock even when scraper throws an error', async () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
let capturedCallback;
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn((expr, cb) => {
capturedCallback = cb;
return { stop: jest.fn() };
})
}));
const freshRunScrape = jest.fn(() => Promise.reject(new Error('Scrape failed')));
jest.doMock('../../services/scraperService', () => ({
runScrape: freshRunScrape
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-error-test')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
freshInit(mockDb);
await capturedCallback();
// Lock should be released even after error
expect(freshIsRunning()).toBe(false);
});
test('should skip run if scraper is already in progress', async () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
let capturedCallback;
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn((expr, cb) => {
capturedCallback = cb;
return { stop: jest.fn() };
})
}));
const freshRunScrape = jest.fn(() => Promise.resolve({ status: 'success' }));
jest.doMock('../../services/scraperService', () => ({
runScrape: freshRunScrape
}));
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-skip-test')
}));
const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
freshInit(mockDb);
// Simulate an already-running scraper
freshAcquire('existing-job');
await capturedCallback();
// runScrape should NOT have been called since lock was already held
expect(freshRunScrape).not.toHaveBeenCalled();
expect(freshLogger.warn).toHaveBeenCalledWith(
expect.stringContaining('already in progress')
);
});
});
describe('stopScheduler()', () => {
test('should stop the scheduled cron job', () => {
jest.resetModules();
const mockStop = jest.fn();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: mockStop }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
}));
const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
freshInit(mockDb);
freshStop();
expect(mockStop).toHaveBeenCalled();
});
test('should be safe to call when no scheduler is active', () => {
expect(() => stopScheduler()).not.toThrow();
});
});
describe('getScheduleExpression()', () => {
test('should return "disabled" when SCRAPER_ENABLED is false', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: false,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
expect(freshGetExpr()).toBe('disabled');
});
test('should return the configured cron expression when enabled', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '30 8 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
expect(freshGetExpr()).toBe('30 8 * * *');
});
});
describe('getNextScheduledRun()', () => {
test('should return null when SCRAPER_ENABLED is false', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: false,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
expect(freshGetNext()).toBeNull();
});
test('should return null when no scheduler is active', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({ validate: jest.fn(), schedule: jest.fn() }));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
// No initializeScheduler called, so no scheduledJob
expect(freshGetNext()).toBeNull();
});
test('should return an ISO date string when scheduler is active', () => {
jest.resetModules();
jest.doMock('../../config/scraper', () => ({
SCRAPER_ENABLED: true,
SCRAPER_SCHEDULE: '0 6 * * *',
SCRAPER_TIMEZONE: 'UTC'
}));
jest.doMock('node-cron', () => ({
validate: jest.fn(() => true),
schedule: jest.fn(() => ({ stop: jest.fn() }))
}));
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
freshInit(mockDb);
const nextRun = freshGetNext();
// Should be a valid ISO date string
expect(nextRun).not.toBeNull();
expect(new Date(nextRun).toISOString()).toBe(nextRun);
});
});
});