feat: add graceful shutdown handling to scraper job scheduler
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 39s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 39s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Add gracefulShutdown(), registerSignalHandlers(), and isShuttingDown() to enable clean process termination during scraper operations. Key changes: - Register SIGTERM and SIGINT signal handlers for process shutdown - Wait for in-progress scraper jobs to complete before exiting, with a configurable timeout (SCRAPER_SHUTDOWN_TIMEOUT, default 30s) - Release mutex lock on shutdown to prevent stale lock state - Track shutdown state to reject new job acquisitions during shutdown - Stop cron scheduler as part of the shutdown sequence - Add SHUTDOWN_TIMEOUT to scraper config module Adds 14 new tests covering shutdown lifecycle, timeout enforcement, idempotent shutdown calls, and lock rejection during shutdown (56 total).
This commit is contained in:
@ -748,6 +748,551 @@ describe('scraperJob - scheduler', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('gracefulShutdown()', () => {
|
||||
test('should stop the cron scheduler', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockStop = jest.fn();
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: mockStop }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
await freshShutdown();
|
||||
|
||||
expect(mockStop).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should release mutex lock if held', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const {
|
||||
acquireLock: freshAcquire,
|
||||
isScraperRunning: freshIsRunning,
|
||||
gracefulShutdown: freshShutdown
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
// Acquire lock to simulate a running job that already completed
|
||||
// (no runningJobPromise set)
|
||||
freshAcquire('shutdown-test-job');
|
||||
expect(freshIsRunning()).toBe(true);
|
||||
|
||||
await freshShutdown();
|
||||
|
||||
// Lock should be released
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should wait for running job to complete before shutdown', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
let resolveRunScrape;
|
||||
const runScrapePromise = new Promise((resolve) => {
|
||||
resolveRunScrape = resolve;
|
||||
});
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => runScrapePromise);
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'running-job-id')
|
||||
}));
|
||||
|
||||
const {
|
||||
initializeScheduler: freshInit,
|
||||
gracefulShutdown: freshShutdown,
|
||||
isScraperRunning: freshIsRunning
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Trigger the cron callback to start a job (don't await - it's still running)
|
||||
const jobPromise = capturedCallback();
|
||||
|
||||
// The job should be running
|
||||
expect(freshIsRunning()).toBe(true);
|
||||
|
||||
// Start shutdown (it should wait for the running job)
|
||||
let shutdownCompleted = false;
|
||||
const shutdownPromise = freshShutdown().then(() => {
|
||||
shutdownCompleted = true;
|
||||
});
|
||||
|
||||
// Give shutdown a tick to start waiting
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
// Shutdown should NOT have completed yet (job still running)
|
||||
expect(shutdownCompleted).toBe(false);
|
||||
|
||||
// Now resolve the running scrape
|
||||
resolveRunScrape({ status: 'success' });
|
||||
await jobPromise;
|
||||
|
||||
// Now shutdown should complete
|
||||
await shutdownPromise;
|
||||
expect(shutdownCompleted).toBe(true);
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
});
|
||||
|
||||
test('should enforce shutdown timeout if job takes too long', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
// Create a promise that never resolves (simulates a hung job)
|
||||
const neverResolvingPromise = new Promise(() => {});
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: true,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 200 // Very short timeout for test
|
||||
}));
|
||||
|
||||
let capturedCallback;
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn((expr, cb) => {
|
||||
capturedCallback = cb;
|
||||
return { stop: jest.fn() };
|
||||
})
|
||||
}));
|
||||
|
||||
const freshRunScrape = jest.fn(() => neverResolvingPromise);
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: freshRunScrape
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'timeout-job-id')
|
||||
}));
|
||||
|
||||
const {
|
||||
initializeScheduler: freshInit,
|
||||
gracefulShutdown: freshShutdown,
|
||||
isScraperRunning: freshIsRunning
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
freshInit(mockDb);
|
||||
|
||||
// Trigger the cron callback to start a job (don't await - it never resolves)
|
||||
capturedCallback();
|
||||
|
||||
expect(freshIsRunning()).toBe(true);
|
||||
|
||||
// Shutdown should complete within timeout even though job is hung
|
||||
await freshShutdown();
|
||||
|
||||
// Lock should be force-released after timeout
|
||||
expect(freshIsRunning()).toBe(false);
|
||||
|
||||
// Should have logged the timeout warning
|
||||
expect(freshLogger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('timed out'),
|
||||
expect.any(Object)
|
||||
);
|
||||
}, 10000);
|
||||
|
||||
test('should log shutdown initiation and completion', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => freshLogger)
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
|
||||
|
||||
await freshShutdown();
|
||||
|
||||
// Should log shutdown initiation
|
||||
expect(freshLogger.info).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Shutdown initiated')
|
||||
);
|
||||
|
||||
// Should log shutdown completion
|
||||
expect(freshLogger.info).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Shutdown complete')
|
||||
);
|
||||
});
|
||||
|
||||
test('should be safe to call multiple times', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
|
||||
|
||||
// Should not throw on multiple calls
|
||||
await freshShutdown();
|
||||
await freshShutdown();
|
||||
});
|
||||
|
||||
test('should prevent new jobs from being accepted after shutdown starts', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(() => true),
|
||||
schedule: jest.fn(() => ({ stop: jest.fn() }))
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const {
|
||||
gracefulShutdown: freshShutdown,
|
||||
acquireLock: freshAcquire,
|
||||
isShuttingDown: freshIsShuttingDown
|
||||
} = require('../../jobs/scraperJob');
|
||||
|
||||
await freshShutdown();
|
||||
|
||||
// After shutdown, acquireLock should return false
|
||||
const lockResult = freshAcquire('post-shutdown-job');
|
||||
expect(lockResult).toBe(false);
|
||||
|
||||
// isShuttingDown should return true
|
||||
expect(freshIsShuttingDown()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('registerSignalHandlers()', () => {
|
||||
test('should export registerSignalHandlers as a function', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(typeof freshRegister).toBe('function');
|
||||
});
|
||||
|
||||
test('should register SIGTERM and SIGINT handlers', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
|
||||
|
||||
// Mock process.on to prevent actual signal handler registration
|
||||
// which would interfere with the test runner
|
||||
const originalOn = process.on.bind(process);
|
||||
const registeredHandlers = [];
|
||||
const onMock = jest.fn((signal, handler) => {
|
||||
registeredHandlers.push({ signal, handler });
|
||||
});
|
||||
process.on = onMock;
|
||||
|
||||
try {
|
||||
freshRegister();
|
||||
|
||||
const registeredSignals = registeredHandlers.map(h => h.signal);
|
||||
expect(registeredSignals).toContain('SIGTERM');
|
||||
expect(registeredSignals).toContain('SIGINT');
|
||||
} finally {
|
||||
process.on = originalOn;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('isShuttingDown()', () => {
|
||||
test('should return false initially', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob');
|
||||
|
||||
expect(freshIsShuttingDown()).toBe(false);
|
||||
});
|
||||
|
||||
test('should return true after gracefulShutdown is called', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
|
||||
|
||||
await freshShutdown();
|
||||
|
||||
expect(freshIsShuttingDown()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('module exports - shutdown functions', () => {
|
||||
test('should export gracefulShutdown as a function', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { gracefulShutdown } = require('../../jobs/scraperJob');
|
||||
expect(typeof gracefulShutdown).toBe('function');
|
||||
});
|
||||
|
||||
test('should export isShuttingDown as a function', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { isShuttingDown } = require('../../jobs/scraperJob');
|
||||
expect(typeof isShuttingDown).toBe('function');
|
||||
});
|
||||
|
||||
test('should export registerSignalHandlers as a function', () => {
|
||||
jest.resetModules();
|
||||
|
||||
jest.doMock('../../config/scraper', () => ({
|
||||
SCRAPER_ENABLED: false,
|
||||
SCRAPER_SCHEDULE: '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: 'UTC',
|
||||
SHUTDOWN_TIMEOUT: 5000
|
||||
}));
|
||||
jest.doMock('node-cron', () => ({
|
||||
validate: jest.fn(), schedule: jest.fn()
|
||||
}));
|
||||
jest.doMock('../../services/scraperLogger', () => ({
|
||||
createLogger: jest.fn(() => ({
|
||||
info: jest.fn(), warn: jest.fn(), error: jest.fn()
|
||||
}))
|
||||
}));
|
||||
jest.doMock('../../services/scraperService', () => ({
|
||||
runScrape: jest.fn()
|
||||
}));
|
||||
jest.doMock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mock-uuid')
|
||||
}));
|
||||
|
||||
const { registerSignalHandlers } = require('../../jobs/scraperJob');
|
||||
expect(typeof registerSignalHandlers).toBe('function');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getScheduleExpression()', () => {
|
||||
test('should return "disabled" when SCRAPER_ENABLED is false', () => {
|
||||
jest.resetModules();
|
||||
|
||||
@ -27,6 +27,9 @@ module.exports = {
|
||||
timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
|
||||
},
|
||||
|
||||
// Graceful shutdown timeout (how long to wait for running job before force-stopping)
|
||||
SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
|
||||
|
||||
// MongoDB collection names (environment variable overrides for development isolation)
|
||||
COLLECTIONS: {
|
||||
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
|
||||
|
||||
@ -11,6 +11,10 @@ let currentJobId = null;
|
||||
// Scheduler state
|
||||
let scheduledJob = null;
|
||||
|
||||
// Shutdown state
|
||||
let shuttingDown = false;
|
||||
let runningJobPromise = null;
|
||||
|
||||
/**
|
||||
* Check if scraper is currently running
|
||||
* @returns {boolean}
|
||||
@ -33,7 +37,7 @@ function getCurrentJobId() {
|
||||
* @returns {boolean} True if lock acquired
|
||||
*/
|
||||
function acquireLock(jobId) {
|
||||
if (isRunning) {
|
||||
if (isRunning || shuttingDown) {
|
||||
return false;
|
||||
}
|
||||
isRunning = true;
|
||||
@ -141,20 +145,23 @@ function initializeScheduler(db) {
|
||||
|
||||
jobLogger.info('Scheduled scrape triggered');
|
||||
|
||||
// Check if already running
|
||||
// Check if already running or shutting down
|
||||
if (!acquireLock(jobId)) {
|
||||
jobLogger.warn('Skipped - scrape already in progress');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await runScrape(db, { trigger: 'scheduled', jobId });
|
||||
const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
|
||||
runningJobPromise = jobExecution;
|
||||
await jobExecution;
|
||||
} catch (error) {
|
||||
jobLogger.error('Scheduled scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
} finally {
|
||||
runningJobPromise = null;
|
||||
releaseLock();
|
||||
}
|
||||
}, {
|
||||
@ -179,6 +186,80 @@ function stopScheduler() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is in the process of shutting down
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isShuttingDown() {
|
||||
return shuttingDown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a graceful shutdown of the scraper
|
||||
* - Stops the cron scheduler to prevent new jobs
|
||||
* - Waits for any running job to complete (with timeout)
|
||||
* - Releases the mutex lock
|
||||
* - Logs shutdown progress
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function gracefulShutdown() {
|
||||
const logger = createLogger('shutdown');
|
||||
|
||||
logger.info('Shutdown initiated');
|
||||
|
||||
// Mark as shutting down to prevent new jobs
|
||||
shuttingDown = true;
|
||||
|
||||
// Stop the cron scheduler
|
||||
stopScheduler();
|
||||
|
||||
// Wait for running job to complete (with timeout)
|
||||
if (isRunning && runningJobPromise) {
|
||||
logger.info('Waiting for running job to complete', {
|
||||
jobId: currentJobId,
|
||||
timeout: config.SHUTDOWN_TIMEOUT
|
||||
});
|
||||
|
||||
let timeoutHandle;
|
||||
const timeoutPromise = new Promise((resolve) => {
|
||||
timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
|
||||
});
|
||||
|
||||
const result = await Promise.race([
|
||||
runningJobPromise.then(() => 'completed').catch(() => 'completed'),
|
||||
timeoutPromise
|
||||
]);
|
||||
|
||||
clearTimeout(timeoutHandle);
|
||||
|
||||
if (result === 'timeout') {
|
||||
logger.warn('Shutdown wait for running job timed out', {
|
||||
jobId: currentJobId,
|
||||
timeout: config.SHUTDOWN_TIMEOUT
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Force release the lock
|
||||
releaseLock();
|
||||
|
||||
logger.info('Shutdown complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Register process signal handlers for graceful shutdown
|
||||
* Listens for SIGTERM and SIGINT signals
|
||||
*/
|
||||
function registerSignalHandlers() {
|
||||
process.on('SIGTERM', () => {
|
||||
gracefulShutdown();
|
||||
});
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
gracefulShutdown();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
@ -187,5 +268,8 @@ module.exports = {
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun,
|
||||
initializeScheduler,
|
||||
stopScheduler
|
||||
stopScheduler,
|
||||
isShuttingDown,
|
||||
gracefulShutdown,
|
||||
registerSignalHandlers
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user