Integrate scraper scheduler and index creation into server.js startup

Add scraper initialization to the connectToMongoDB() startup flow:
- Import createScraperIndexes, initializeScheduler, and
  registerSignalHandlers from their respective service modules
- Create scraper database indexes during MongoDB connection setup
- Conditionally start the cron scheduler when SCRAPER_ENABLED is true
- Register SIGTERM/SIGINT signal handlers for graceful shutdown

All scraper initialization is wrapped in try/catch so failures are
logged but do not prevent the server from starting. This ensures
existing API functionality remains unaffected even if scraper
components encounter errors during startup.

Add 11 integration tests covering startup behavior, error resilience,
SCRAPER_ENABLED guard, and signal handler registration.

__tests__/scraper/serverIntegration.test.js

@@ -0,0 +1,177 @@
+/**
+ * Tests for scraper integration in server.js startup
+ *
+ * Covers:
+ * - Server initializes scraper scheduler on startup
+ * - Server does not crash if scheduler init fails
+ * - Indexes are created before scheduler starts
+ * - Scheduler not initialized when SCRAPER_ENABLED=false
+ *
+ * Strategy: Instead of importing server.js directly (which starts Express and
+ * connects to MongoDB), we replicate the scraper initialization logic from
+ * connectToMongoDB() and test it with mocked dependencies. This verifies the
+ * integration contract without needing a full server boot.
+ */
+
+const { MongoClient } = require('mongodb');
+
+describe('Server Scraper Integration', () => {
+  let connection;
+  let db;
+
+  beforeAll(async () => {
+    const uri = process.env.MONGO_URI;
+    connection = await MongoClient.connect(uri);
+    db = connection.db('apartments_server_integration_test');
+  });
+
+  afterAll(async () => {
+    if (connection) {
+      await connection.close();
+    }
+  });
+
+  describe('scraper initialization during startup', () => {
+    let mockCreateScraperIndexes;
+    let mockInitializeScheduler;
+    let mockCreateLogger;
+    let mockLogger;
+
+    beforeEach(() => {
+      mockLogger = {
+        info: jest.fn(),
+        warn: jest.fn(),
+        error: jest.fn(),
+        debug: jest.fn()
+      };
+      mockCreateLogger = jest.fn(() => mockLogger);
+      mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
+      mockInitializeScheduler = jest.fn();
+    });
+
+    /**
+     * Replicates the scraper initialization logic from server.js connectToMongoDB().
+     * This is the exact pattern used in production:
+     *
+     *   try {
+     *     const scraperLogger = createLogger('server-init');
+     *     await createScraperIndexes(db, scraperLogger);
+     *     if (scraperConfig.SCRAPER_ENABLED) {
+     *       initializeScheduler(db);
+     *     }
+     *   } catch (error) {
+     *     console.error('Scraper initialization failed (server continues):', error.message);
+     *   }
+     */
+    async function runScraperInit(testDb, config) {
+      try {
+        const scraperLogger = mockCreateLogger('server-init');
+        await mockCreateScraperIndexes(testDb, scraperLogger);
+        if (config.SCRAPER_ENABLED) {
+          mockInitializeScheduler(testDb);
+        }
+        return { success: true };
+      } catch (error) {
+        return { success: false, error: error.message };
+      }
+    }
+
+    it('should initialize scraper scheduler on startup when enabled', async () => {
+      const config = { SCRAPER_ENABLED: true };
+
+      const result = await runScraperInit(db, config);
+
+      expect(result.success).toBe(true);
+      expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
+      expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
+    });
+
+    it('should not crash if createScraperIndexes throws an error', async () => {
+      mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
+      const config = { SCRAPER_ENABLED: true };
+
+      const result = await runScraperInit(db, config);
+
+      // The server should continue (error is caught, not thrown)
+      expect(result.success).toBe(false);
+      expect(result.error).toBe('Index creation failed');
+      // initializeScheduler should NOT have been called since the error occurred before it
+      expect(mockInitializeScheduler).not.toHaveBeenCalled();
+    });
+
+    it('should create indexes before initializing the scheduler', async () => {
+      const callOrder = [];
+      mockCreateScraperIndexes.mockImplementation(async () => {
+        callOrder.push('createScraperIndexes');
+      });
+      mockInitializeScheduler.mockImplementation(() => {
+        callOrder.push('initializeScheduler');
+      });
+
+      const config = { SCRAPER_ENABLED: true };
+      await runScraperInit(db, config);
+
+      expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
+    });
+
+    it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
+      const config = { SCRAPER_ENABLED: false };
+
+      const result = await runScraperInit(db, config);
+
+      expect(result.success).toBe(true);
+      expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
+      expect(mockInitializeScheduler).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('server.js imports and exports verification', () => {
+    it('should be able to import createScraperIndexes from scraperService', () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      expect(typeof createScraperIndexes).toBe('function');
+    });
+
+    it('should be able to import initializeScheduler from scraperJob', () => {
+      const { initializeScheduler } = require('../../jobs/scraperJob');
+      expect(typeof initializeScheduler).toBe('function');
+    });
+
+    it('should be able to import registerSignalHandlers from scraperJob', () => {
+      const { registerSignalHandlers } = require('../../jobs/scraperJob');
+      expect(typeof registerSignalHandlers).toBe('function');
+    });
+
+    it('should be able to import SCRAPER_ENABLED from scraper config', () => {
+      const scraperConfig = require('../../config/scraper');
+      expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
+    });
+  });
+
+  describe('createScraperIndexes error handling with real db', () => {
+    it('should handle index creation gracefully with a valid db', async () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      const { createLogger } = require('../../services/scraperLogger');
+      const logger = createLogger('test-server-init');
+
+      // Should not throw - creates indexes on the test db
+      await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
+    });
+
+    it('should throw when given an invalid db object', async () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
+
+      // Passing null should throw (simulates what the try/catch in server.js protects against)
+      await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
+    });
+  });
+
+  describe('registerSignalHandlers', () => {
+    it('should register SIGTERM and SIGINT handlers without throwing', () => {
+      const { registerSignalHandlers } = require('../../jobs/scraperJob');
+
+      // Should not throw when called
+      expect(() => registerSignalHandlers()).not.toThrow();
+    });
+  });
+});

server.js

@@ -11,6 +11,9 @@ const activityRoutes = require('./routes/activity');
 const adminRoutes = require('./routes/admin');
 const { createIndexes } = require('./models/user');
 const { createActivityIndexes } = require('./services/activityLogger');
+const { createScraperIndexes } = require('./services/scraperService');
+const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
+const scraperConfig = require('./config/scraper');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
@@ -80,12 +83,30 @@ async function connectToMongoDB() {
     await createIndexes(db);
     await createActivityIndexes(db);
     console.log('✅ Passport configured and indexes created');
+
+    // Initialize scraper (indexes + scheduler) - errors logged but don't crash server
+    try {
+      const { createLogger } = require('./services/scraperLogger');
+      const scraperLogger = createLogger('server-init');
+      await createScraperIndexes(db, scraperLogger);
+      if (scraperConfig.SCRAPER_ENABLED) {
+        initializeScheduler(db);
+        console.log('✅ Scraper scheduler initialized');
+      } else {
+        console.log('ℹ️ Scraper scheduling disabled');
+      }
+    } catch (error) {
+      console.error('⚠️ Scraper initialization failed (server continues):', error.message);
+    }
   } catch (error) {
     console.error('❌ Failed to connect to MongoDB:', error);
     process.exit(1);
   }
 }
 
+// Register scraper signal handlers for graceful shutdown
+registerSignalHandlers();
+
 // Helper function to get today's date in UTC
 const getTodayDateUTC = () => new Date().toISOString().split('T')[0];