SCRAPE-14: Add graceful shutdown handling (#19)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

__tests__/scraper/scraperJob.test.js

@@ -748,6 +748,551 @@ describe('scraperJob - scheduler', () => {
     });
   });
 
+  describe('gracefulShutdown()', () => {
+    test('should stop the cron scheduler', async () => {
+      jest.resetModules();
+
+      const mockStop = jest.fn();
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: true,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn(() => ({ stop: mockStop }))
+      }));
+      const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => freshLogger)
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
+
+      freshInit(mockDb);
+      await freshShutdown();
+
+      expect(mockStop).toHaveBeenCalled();
+    });
+
+    test('should release mutex lock if held', async () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn(() => ({ stop: jest.fn() }))
+      }));
+      const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => freshLogger)
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const {
+        acquireLock: freshAcquire,
+        isScraperRunning: freshIsRunning,
+        gracefulShutdown: freshShutdown
+      } = require('../../jobs/scraperJob');
+
+      // Acquire lock to simulate a running job that already completed
+      // (no runningJobPromise set)
+      freshAcquire('shutdown-test-job');
+      expect(freshIsRunning()).toBe(true);
+
+      await freshShutdown();
+
+      // Lock should be released
+      expect(freshIsRunning()).toBe(false);
+    });
+
+    test('should wait for running job to complete before shutdown', async () => {
+      jest.resetModules();
+
+      let resolveRunScrape;
+      const runScrapePromise = new Promise((resolve) => {
+        resolveRunScrape = resolve;
+      });
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: true,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+
+      let capturedCallback;
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn((expr, cb) => {
+          capturedCallback = cb;
+          return { stop: jest.fn() };
+        })
+      }));
+
+      const freshRunScrape = jest.fn(() => runScrapePromise);
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: freshRunScrape
+      }));
+      const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => freshLogger)
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'running-job-id')
+      }));
+
+      const {
+        initializeScheduler: freshInit,
+        gracefulShutdown: freshShutdown,
+        isScraperRunning: freshIsRunning
+      } = require('../../jobs/scraperJob');
+
+      freshInit(mockDb);
+
+      // Trigger the cron callback to start a job (don't await - it's still running)
+      const jobPromise = capturedCallback();
+
+      // The job should be running
+      expect(freshIsRunning()).toBe(true);
+
+      // Start shutdown (it should wait for the running job)
+      let shutdownCompleted = false;
+      const shutdownPromise = freshShutdown().then(() => {
+        shutdownCompleted = true;
+      });
+
+      // Give shutdown a tick to start waiting
+      await new Promise(resolve => setTimeout(resolve, 50));
+
+      // Shutdown should NOT have completed yet (job still running)
+      expect(shutdownCompleted).toBe(false);
+
+      // Now resolve the running scrape
+      resolveRunScrape({ status: 'success' });
+      await jobPromise;
+
+      // Now shutdown should complete
+      await shutdownPromise;
+      expect(shutdownCompleted).toBe(true);
+      expect(freshIsRunning()).toBe(false);
+    });
+
+    test('should enforce shutdown timeout if job takes too long', async () => {
+      jest.resetModules();
+
+      // Create a promise that never resolves (simulates a hung job)
+      const neverResolvingPromise = new Promise(() => {});
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: true,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 200 // Very short timeout for test
+      }));
+
+      let capturedCallback;
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn((expr, cb) => {
+          capturedCallback = cb;
+          return { stop: jest.fn() };
+        })
+      }));
+
+      const freshRunScrape = jest.fn(() => neverResolvingPromise);
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: freshRunScrape
+      }));
+      const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => freshLogger)
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'timeout-job-id')
+      }));
+
+      const {
+        initializeScheduler: freshInit,
+        gracefulShutdown: freshShutdown,
+        isScraperRunning: freshIsRunning
+      } = require('../../jobs/scraperJob');
+
+      freshInit(mockDb);
+
+      // Trigger the cron callback to start a job (don't await - it never resolves)
+      capturedCallback();
+
+      expect(freshIsRunning()).toBe(true);
+
+      // Shutdown should complete within timeout even though job is hung
+      await freshShutdown();
+
+      // Lock should be force-released after timeout
+      expect(freshIsRunning()).toBe(false);
+
+      // Should have logged the timeout warning
+      expect(freshLogger.warn).toHaveBeenCalledWith(
+        expect.stringContaining('timed out'),
+        expect.any(Object)
+      );
+    }, 10000);
+
+    test('should log shutdown initiation and completion', async () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn(() => ({ stop: jest.fn() }))
+      }));
+      const freshLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => freshLogger)
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
+
+      await freshShutdown();
+
+      // Should log shutdown initiation
+      expect(freshLogger.info).toHaveBeenCalledWith(
+        expect.stringContaining('Shutdown initiated')
+      );
+
+      // Should log shutdown completion
+      expect(freshLogger.info).toHaveBeenCalledWith(
+        expect.stringContaining('Shutdown complete')
+      );
+    });
+
+    test('should be safe to call multiple times', async () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn(() => ({ stop: jest.fn() }))
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
+
+      // Should not throw on multiple calls
+      await freshShutdown();
+      await freshShutdown();
+    });
+
+    test('should prevent new jobs from being accepted after shutdown starts', async () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(() => true),
+        schedule: jest.fn(() => ({ stop: jest.fn() }))
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const {
+        gracefulShutdown: freshShutdown,
+        acquireLock: freshAcquire,
+        isShuttingDown: freshIsShuttingDown
+      } = require('../../jobs/scraperJob');
+
+      await freshShutdown();
+
+      // After shutdown, acquireLock should return false
+      const lockResult = freshAcquire('post-shutdown-job');
+      expect(lockResult).toBe(false);
+
+      // isShuttingDown should return true
+      expect(freshIsShuttingDown()).toBe(true);
+    });
+  });
+
+  describe('registerSignalHandlers()', () => {
+    test('should export registerSignalHandlers as a function', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
+
+      expect(typeof freshRegister).toBe('function');
+    });
+
+    test('should register SIGTERM and SIGINT handlers', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
+
+      // Mock process.on to prevent actual signal handler registration
+      // which would interfere with the test runner
+      const originalOn = process.on.bind(process);
+      const registeredHandlers = [];
+      const onMock = jest.fn((signal, handler) => {
+        registeredHandlers.push({ signal, handler });
+      });
+      process.on = onMock;
+
+      try {
+        freshRegister();
+
+        const registeredSignals = registeredHandlers.map(h => h.signal);
+        expect(registeredSignals).toContain('SIGTERM');
+        expect(registeredSignals).toContain('SIGINT');
+      } finally {
+        process.on = originalOn;
+      }
+    });
+  });
+
+  describe('isShuttingDown()', () => {
+    test('should return false initially', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob');
+
+      expect(freshIsShuttingDown()).toBe(false);
+    });
+
+    test('should return true after gracefulShutdown is called', async () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
+
+      await freshShutdown();
+
+      expect(freshIsShuttingDown()).toBe(true);
+    });
+  });
+
+  describe('module exports - shutdown functions', () => {
+    test('should export gracefulShutdown as a function', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { gracefulShutdown } = require('../../jobs/scraperJob');
+      expect(typeof gracefulShutdown).toBe('function');
+    });
+
+    test('should export isShuttingDown as a function', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { isShuttingDown } = require('../../jobs/scraperJob');
+      expect(typeof isShuttingDown).toBe('function');
+    });
+
+    test('should export registerSignalHandlers as a function', () => {
+      jest.resetModules();
+
+      jest.doMock('../../config/scraper', () => ({
+        SCRAPER_ENABLED: false,
+        SCRAPER_SCHEDULE: '0 6 * * *',
+        SCRAPER_TIMEZONE: 'UTC',
+        SHUTDOWN_TIMEOUT: 5000
+      }));
+      jest.doMock('node-cron', () => ({
+        validate: jest.fn(), schedule: jest.fn()
+      }));
+      jest.doMock('../../services/scraperLogger', () => ({
+        createLogger: jest.fn(() => ({
+          info: jest.fn(), warn: jest.fn(), error: jest.fn()
+        }))
+      }));
+      jest.doMock('../../services/scraperService', () => ({
+        runScrape: jest.fn()
+      }));
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
+      }));
+
+      const { registerSignalHandlers } = require('../../jobs/scraperJob');
+      expect(typeof registerSignalHandlers).toBe('function');
+    });
+  });
+
   describe('getScheduleExpression()', () => {
     test('should return "disabled" when SCRAPER_ENABLED is false', () => {
       jest.resetModules();

config/scraper.js

@@ -27,6 +27,9 @@ module.exports = {
     timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
   },
 
+  // Graceful shutdown timeout (how long to wait for running job before force-stopping)
+  SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
+
   // MongoDB collection names (environment variable overrides for development isolation)
   COLLECTIONS: {
     UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',

jobs/scraperJob.js

@@ -11,6 +11,10 @@ let currentJobId = null;
 // Scheduler state
 let scheduledJob = null;
 
+// Shutdown state
+let shuttingDown = false;
+let runningJobPromise = null;
+
 /**
  * Check if scraper is currently running
  * @returns {boolean}
@@ -33,7 +37,7 @@ function getCurrentJobId() {
  * @returns {boolean} True if lock acquired
  */
 function acquireLock(jobId) {
-  if (isRunning) {
+  if (isRunning || shuttingDown) {
     return false;
   }
   isRunning = true;
@@ -141,20 +145,23 @@ function initializeScheduler(db) {
 
     jobLogger.info('Scheduled scrape triggered');
 
-    // Check if already running
+    // Check if already running or shutting down
     if (!acquireLock(jobId)) {
       jobLogger.warn('Skipped - scrape already in progress');
       return;
     }
 
     try {
-      await runScrape(db, { trigger: 'scheduled', jobId });
+      const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
+      runningJobPromise = jobExecution;
+      await jobExecution;
     } catch (error) {
       jobLogger.error('Scheduled scrape failed', {
         errorType: error.name,
         errorMessage: error.message
       });
     } finally {
+      runningJobPromise = null;
       releaseLock();
     }
   }, {
@@ -179,6 +186,80 @@ function stopScheduler() {
   }
 }
 
+/**
+ * Check if the scraper is in the process of shutting down
+ * @returns {boolean}
+ */
+function isShuttingDown() {
+  return shuttingDown;
+}
+
+/**
+ * Perform a graceful shutdown of the scraper
+ * - Stops the cron scheduler to prevent new jobs
+ * - Waits for any running job to complete (with timeout)
+ * - Releases the mutex lock
+ * - Logs shutdown progress
+ * @returns {Promise<void>}
+ */
+async function gracefulShutdown() {
+  const logger = createLogger('shutdown');
+
+  logger.info('Shutdown initiated');
+
+  // Mark as shutting down to prevent new jobs
+  shuttingDown = true;
+
+  // Stop the cron scheduler
+  stopScheduler();
+
+  // Wait for running job to complete (with timeout)
+  if (isRunning && runningJobPromise) {
+    logger.info('Waiting for running job to complete', {
+      jobId: currentJobId,
+      timeout: config.SHUTDOWN_TIMEOUT
+    });
+
+    let timeoutHandle;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
+    });
+
+    const result = await Promise.race([
+      runningJobPromise.then(() => 'completed').catch(() => 'completed'),
+      timeoutPromise
+    ]);
+
+    clearTimeout(timeoutHandle);
+
+    if (result === 'timeout') {
+      logger.warn('Shutdown wait for running job timed out', {
+        jobId: currentJobId,
+        timeout: config.SHUTDOWN_TIMEOUT
+      });
+    }
+  }
+
+  // Force release the lock
+  releaseLock();
+
+  logger.info('Shutdown complete');
+}
+
+/**
+ * Register process signal handlers for graceful shutdown
+ * Listens for SIGTERM and SIGINT signals
+ */
+function registerSignalHandlers() {
+  process.on('SIGTERM', () => {
+    gracefulShutdown();
+  });
+
+  process.on('SIGINT', () => {
+    gracefulShutdown();
+  });
+}
+
 module.exports = {
   isScraperRunning,
   getCurrentJobId,
@@ -187,5 +268,8 @@ module.exports = {
   getScheduleExpression,
   getNextScheduledRun,
   initializeScheduler,
-  stopScheduler
+  stopScheduler,
+  isShuttingDown,
+  gracefulShutdown,
+  registerSignalHandlers
 };