feat: add GET /api/admin/scraper/status endpoint

Implement a status endpoint that returns the current state of the
scraper system. The response includes mutex state (idle/running with
job ID), last run history from scraper_runs collection (status,
timing, unit/error counts), next scheduled run timestamp, and
cron schedule expression.

Protected by requireAuth + requireAdmin middleware. Returns 503
on database errors for graceful degradation. Includes 13 tests
covering auth, response structure, edge cases, and error handling.

__tests__/scraper/scraperJob.test.js

@@ -18,10 +18,9 @@ jest.mock('../../services/scraperLogger', () => ({
   }))
 }));
 
-// Mock crypto.randomUUID
-jest.mock('crypto', () => ({
-  ...jest.requireActual('crypto'),
-  randomUUID: jest.fn(() => 'mock-uuid-1234')
+// Mock uuid
+jest.mock('uuid', () => ({
+  v4: jest.fn(() => 'mock-uuid-1234')
 }));
 
 // Mock scraperService
@@ -257,9 +256,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -295,9 +293,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -328,9 +325,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -373,9 +369,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -415,9 +410,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -452,9 +446,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -488,9 +481,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
@@ -526,9 +518,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const freshCron = require('node-cron');
@@ -573,9 +564,8 @@ describe('scraperJob - scheduler', () => {
           info: jest.fn(), warn: jest.fn(), error: jest.fn()
         }))
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'scheduled-job-id')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'scheduled-job-id')
       }));
 
       const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@@ -619,9 +609,8 @@ describe('scraperJob - scheduler', () => {
           info: jest.fn(), warn: jest.fn(), error: jest.fn()
         }))
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'job-release-test')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'job-release-test')
       }));
 
       const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@@ -660,9 +649,8 @@ describe('scraperJob - scheduler', () => {
           info: jest.fn(), warn: jest.fn(), error: jest.fn()
         }))
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'job-error-test')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'job-error-test')
       }));
 
       const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@@ -700,9 +688,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperLogger', () => ({
         createLogger: jest.fn(() => freshLogger)
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'job-skip-test')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'job-skip-test')
       }));
 
       const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
@@ -744,9 +731,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
@@ -784,9 +770,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@@ -817,9 +802,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const {
@@ -871,9 +855,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperLogger', () => ({
         createLogger: jest.fn(() => freshLogger)
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'running-job-id')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'running-job-id')
       }));
 
       const {
@@ -942,9 +925,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperLogger', () => ({
         createLogger: jest.fn(() => freshLogger)
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'timeout-job-id')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'timeout-job-id')
       }));
 
       const {
@@ -993,9 +975,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@@ -1034,9 +1015,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@@ -1067,9 +1047,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const {
@@ -1110,9 +1089,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@@ -1140,9 +1118,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@@ -1189,9 +1166,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob');
@@ -1219,9 +1195,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@@ -1253,9 +1228,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { gracefulShutdown } = require('../../jobs/scraperJob');
@@ -1282,9 +1256,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { isShuttingDown } = require('../../jobs/scraperJob');
@@ -1311,9 +1284,8 @@ describe('scraperJob - scheduler', () => {
       jest.doMock('../../services/scraperService', () => ({
         runScrape: jest.fn()
       }));
-      jest.doMock('crypto', () => ({
-        ...jest.requireActual('crypto'),
-        randomUUID: jest.fn(() => 'mock-uuid')
+      jest.doMock('uuid', () => ({
+        v4: jest.fn(() => 'mock-uuid')
       }));
 
       const { registerSignalHandlers } = require('../../jobs/scraperJob');
@@ -1335,7 +1307,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
 
@@ -1355,7 +1327,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
 
@@ -1377,7 +1349,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1397,7 +1369,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1421,7 +1393,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1449,7 +1421,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1479,7 +1451,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1504,7 +1476,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: utcInit, getNextScheduledRun: utcGetNext } = require('../../jobs/scraperJob');
 
@@ -1535,7 +1507,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 
@@ -1567,7 +1539,7 @@ describe('scraperJob - scheduler', () => {
         createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
       }));
       jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
-      jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
+      jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
       const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
 

__tests__/scraper/scraperRoutes.test.js

@@ -1,23 +1,13 @@
 /**
  * Tests for scraper admin route endpoints
  *
- * Covers:
- * - POST /api/admin/scraper/run
- *   - Authentication and authorization (401, 403)
- *   - Successful scrape trigger (202 with jobId)
- *   - Conflict when scraper already running (409)
- *   - Mutex lock acquisition and release
- *   - Activity logging for admin trigger
- *   - dryRun option support
- *   - htmlContent option support
- *   - Response structure validation
- * - GET /api/admin/scraper/status
- *   - Authentication and authorization (401/403)
- *   - Correct status response structure
- *   - Idle vs running state
- *   - lastRun details from database
- *   - nextScheduledRun and schedule fields
- *   - 503 on database errors
+ * Tests GET /api/admin/scraper/status endpoint for:
+ * - Authentication and authorization (401/403)
+ * - Correct status response structure
+ * - Idle vs running state
+ * - lastRun details from database
+ * - nextScheduledRun and schedule fields
+ * - 503 on database errors
  */
 
 const request = require('supertest');
@@ -40,29 +30,14 @@ jest.mock('../../jobs/scraperJob', () => {
 
   return {
     isScraperRunning: jest.fn(() => _isRunning),
-    acquireLock: jest.fn((jobId) => {
-      if (_isRunning) return false;
-      _isRunning = true;
-      _currentJobId = jobId;
-      return true;
-    }),
-    releaseLock: jest.fn(() => {
-      _isRunning = false;
-      _currentJobId = null;
-    }),
     getCurrentJobId: jest.fn(() => _currentJobId),
     getScheduleExpression: jest.fn(() => _scheduleExpression),
     getNextScheduledRun: jest.fn(() => _nextScheduledRun),
+    acquireLock: jest.fn(),
+    releaseLock: jest.fn(),
     initializeScheduler: jest.fn(),
     stopScheduler: jest.fn(),
-    isShuttingDown: jest.fn(() => false),
-    gracefulShutdown: jest.fn(),
-    registerSignalHandlers: jest.fn(),
     // Test helpers to control mock state
-    _setState: (running, jobId) => {
-      _isRunning = running;
-      _currentJobId = jobId || null;
-    },
     __setRunning: (running, jobId) => {
       _isRunning = running;
       _currentJobId = jobId || null;
@@ -80,20 +55,6 @@ jest.mock('../../jobs/scraperJob', () => {
   };
 });
 
-// Mock scraperService to prevent actual scraping
-jest.mock('../../services/scraperService', () => ({
-  runScrape: jest.fn(() => Promise.resolve({ status: 'success', jobId: 'test-job-id' }))
-}));
-
-// Mock activityLogger to track calls
-jest.mock('../../services/activityLogger', () => {
-  const originalModule = jest.requireActual('../../services/activityLogger');
-  return {
-    ...originalModule,
-    logActivity: jest.fn(() => Promise.resolve())
-  };
-});
-
 // Mock the scraper config for collection names
 jest.mock('../../config/scraper', () => ({
   TARGET_URL: 'https://example.com/test',
@@ -113,8 +74,6 @@ jest.mock('../../config/scraper', () => ({
 }));
 
 const scraperJob = require('../../jobs/scraperJob');
-const { runScrape } = require('../../services/scraperService');
-const { logActivity } = require('../../services/activityLogger');
 
 describe('Scraper Routes', () => {
   let connection;
@@ -124,12 +83,10 @@ describe('Scraper Routes', () => {
   beforeAll(async () => {
     const uri = process.env.MONGO_URI;
     connection = await MongoClient.connect(uri);
-    db = connection.db('apartments_scraper_routes_test');
+    db = connection.db('apartments_test');
   });
 
   afterAll(async () => {
-    // Allow pending async operations (e.g., .finally() callbacks) to complete
-    await new Promise(resolve => setTimeout(resolve, 200));
     if (connection) {
       await connection.close();
     }
@@ -140,438 +97,7 @@ describe('Scraper Routes', () => {
     // Also clean scraper_runs collection
     await db.collection('scraper_runs').deleteMany({});
     app = await createTestApp(db);
-
-    // Reset scraper state
     scraperJob.__reset();
-
-    // Clear mock call history (but keep implementations intact)
-    scraperJob.isScraperRunning.mockClear();
-    scraperJob.acquireLock.mockClear();
-    scraperJob.releaseLock.mockClear();
-    runScrape.mockClear();
-    logActivity.mockClear();
-
-    // Restore default mock implementations
-    runScrape.mockImplementation(() => Promise.resolve({ status: 'success', jobId: 'test-job-id' }));
-  });
-
-  // ============================================================
-  // POST /api/admin/scraper/run
-  // ============================================================
-  describe('POST /api/admin/scraper/run', () => {
-
-    // ============================================================
-    // Authentication Tests
-    // ============================================================
-    describe('authentication and authorization', () => {
-      it('should return 401 without auth token', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .send({});
-
-        expect(res.status).toBe(401);
-        expect(res.body).toHaveProperty('error');
-      });
-
-      it('should return 403 for non-admin user', async () => {
-        const regularUser = createTestUser({ role: 'user' });
-        await insertTestUser(db, regularUser);
-        const token = generateTestToken(regularUser._id);
-
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${token}`])
-          .send({});
-
-        expect(res.status).toBe(403);
-        expect(res.body).toHaveProperty('error');
-        expect(res.body.error).toMatch(/admin/i);
-      });
-    });
-
-    // ============================================================
-    // Successful Trigger Tests
-    // ============================================================
-    describe('successful scrape trigger', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should return 202 with jobId for admin user', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-        expect(res.body).toHaveProperty('data');
-        expect(res.body.data).toHaveProperty('jobId');
-        expect(res.body.data).toHaveProperty('status', 'started');
-        expect(res.body.data).toHaveProperty('message');
-      });
-
-      it('should return response with correct structure (jobId, status, message, dryRun)', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-
-        const { data } = res.body;
-        expect(data.jobId).toBeDefined();
-        expect(typeof data.jobId).toBe('string');
-        expect(data.jobId.length).toBeGreaterThan(0);
-        expect(data.status).toBe('started');
-        expect(typeof data.message).toBe('string');
-        expect(data.dryRun).toBe(false);
-      });
-
-      it('should return a UUID-formatted jobId', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-
-        // UUID v4 format: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx
-        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-        expect(res.body.data.jobId).toMatch(uuidRegex);
-      });
-    });
-
-    // ============================================================
-    // Conflict Tests (409)
-    // ============================================================
-    describe('scraper already running', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should return 409 when scraper is already running', async () => {
-        // Set scraper as running
-        scraperJob._setState(true, 'existing-job-id');
-
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(409);
-        expect(res.body).toHaveProperty('error');
-        expect(res.body.error).toMatch(/already in progress/i);
-      });
-
-      it('should check isScraperRunning before starting', async () => {
-        await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(scraperJob.isScraperRunning).toHaveBeenCalled();
-      });
-    });
-
-    // ============================================================
-    // Lock Acquisition Tests
-    // ============================================================
-    describe('mutex lock behavior', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should acquire lock before starting scrape', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-        expect(scraperJob.acquireLock).toHaveBeenCalledWith(expect.any(String));
-      });
-
-      it('should release lock after scrape completes', async () => {
-        // Make runScrape resolve quickly
-        runScrape.mockImplementation(() => Promise.resolve({ status: 'success' }));
-
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-
-        // Wait for the async scrape to complete
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        // The lock should be released via the .finally() callback
-        expect(scraperJob.releaseLock).toHaveBeenCalled();
-      });
-
-      it('should release lock even when scrape fails', async () => {
-        // Make runScrape reject
-        runScrape.mockImplementation(() => Promise.reject(new Error('Scrape failed')));
-
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-
-        // Wait for the async scrape to complete
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        // Lock should still be released in finally block
-        expect(scraperJob.releaseLock).toHaveBeenCalled();
-      });
-    });
-
-    // ============================================================
-    // Activity Logging Tests
-    // ============================================================
-    describe('activity logging', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should log ADMIN_TRIGGER_SCRAPE activity', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-        expect(logActivity).toHaveBeenCalledWith(
-          expect.anything(), // db
-          expect.objectContaining({
-            userId: adminUser._id.toString(),
-            action: 'ADMIN_TRIGGER_SCRAPE',
-            metadata: expect.objectContaining({
-              jobId: expect.any(String),
-              dryRun: false
-            })
-          })
-        );
-      });
-
-      it('should log dryRun flag in activity metadata', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({ dryRun: true });
-
-        expect(res.status).toBe(202);
-        expect(logActivity).toHaveBeenCalledWith(
-          expect.anything(),
-          expect.objectContaining({
-            action: 'ADMIN_TRIGGER_SCRAPE',
-            metadata: expect.objectContaining({
-              dryRun: true
-            })
-          })
-        );
-      });
-    });
-
-    // ============================================================
-    // dryRun Option Tests
-    // ============================================================
-    describe('dryRun option', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should handle dryRun option in request body', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({ dryRun: true });
-
-        expect(res.status).toBe(202);
-        expect(res.body.data.dryRun).toBe(true);
-        expect(res.body.data.message).toMatch(/dry run/i);
-      });
-
-      it('should pass dryRun option to runScrape', async () => {
-        await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({ dryRun: true });
-
-        // Wait for async execution
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        expect(runScrape).toHaveBeenCalledWith(
-          expect.anything(), // db
-          expect.objectContaining({
-            dryRun: true
-          })
-        );
-      });
-
-      it('should default dryRun to false when not provided', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        expect(res.status).toBe(202);
-        expect(res.body.data.dryRun).toBe(false);
-      });
-    });
-
-    // ============================================================
-    // htmlContent Option Tests
-    // ============================================================
-    describe('htmlContent option', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should pass htmlContent option to runScrape', async () => {
-        const testHtml = '<html><body>Test content</body></html>';
-
-        await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({ htmlContent: testHtml });
-
-        // Wait for async execution
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        expect(runScrape).toHaveBeenCalledWith(
-          expect.anything(),
-          expect.objectContaining({
-            htmlContent: testHtml
-          })
-        );
-      });
-
-      it('should log usingProvidedHtml in activity metadata when htmlContent provided', async () => {
-        const testHtml = '<html><body>Test</body></html>';
-
-        await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({ htmlContent: testHtml });
-
-        expect(logActivity).toHaveBeenCalledWith(
-          expect.anything(),
-          expect.objectContaining({
-            metadata: expect.objectContaining({
-              usingProvidedHtml: true
-            })
-          })
-        );
-      });
-    });
-
-    // ============================================================
-    // runScrape Integration Tests
-    // ============================================================
-    describe('scrape execution', () => {
-      let adminUser;
-      let adminToken;
-
-      beforeEach(async () => {
-        adminUser = createTestAdmin();
-        await insertTestUser(db, adminUser);
-        adminToken = generateTestToken(adminUser._id);
-      });
-
-      it('should call runScrape with trigger "manual"', async () => {
-        await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        // Wait for async execution
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        expect(runScrape).toHaveBeenCalledWith(
-          expect.anything(),
-          expect.objectContaining({
-            trigger: 'manual'
-          })
-        );
-      });
-
-      it('should call runScrape with the generated jobId', async () => {
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        const { jobId } = res.body.data;
-
-        // Wait for async execution
-        await new Promise(resolve => setTimeout(resolve, 100));
-
-        expect(runScrape).toHaveBeenCalledWith(
-          expect.anything(),
-          expect.objectContaining({
-            jobId
-          })
-        );
-      });
-
-      it('should return 202 immediately without waiting for scrape to finish', async () => {
-        // Make runScrape take a "long time" (500ms is enough to prove async behavior)
-        let scrapeResolve;
-        runScrape.mockImplementation(() => new Promise(resolve => {
-          scrapeResolve = resolve;
-          setTimeout(() => resolve({ status: 'success' }), 500);
-        }));
-
-        const startTime = Date.now();
-        const res = await request(app)
-          .post('/api/admin/scraper/run')
-          .set('Cookie', [`auth_token=${adminToken}`])
-          .send({});
-
-        const elapsed = Date.now() - startTime;
-
-        // Should respond in well under 200ms despite scrape taking 500ms
-        expect(res.status).toBe(202);
-        expect(elapsed).toBeLessThan(200);
-
-        // Resolve the scrape to clean up
-        if (scrapeResolve) scrapeResolve({ status: 'success' });
-        await new Promise(resolve => setTimeout(resolve, 100));
-      });
-    });
   });
 
   // ============================================================

jobs/scraperJob.js

@@ -1,8 +1,8 @@
 const cron = require('node-cron');
-const crypto = require('crypto');
 const config = require('../config/scraper');
 const { runScrape } = require('../services/scraperService');
 const { createLogger } = require('../services/scraperLogger');
+const { v4: uuidv4 } = require('uuid');
 
 // In-process mutex state
 let isRunning = false;
@@ -140,7 +140,7 @@ function initializeScheduler(db) {
 
   // Create the scheduled job
   scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
-    const jobId = crypto.randomUUID();
+    const jobId = uuidv4();
     const jobLogger = createLogger(jobId);
 
     jobLogger.info('Scheduled scrape triggered');

routes/admin.js

@@ -1146,70 +1146,14 @@ router.patch('/settings', async (req, res) => {
 // Scraper Endpoints
 // ============================================================
 
-const crypto = require('crypto');
 const {
   isScraperRunning,
-  acquireLock,
-  releaseLock,
   getCurrentJobId,
   getScheduleExpression,
   getNextScheduledRun
 } = require('../jobs/scraperJob');
-const { runScrape } = require('../services/scraperService');
 const scraperConfig = require('../config/scraper');
 
-/**
- * POST /api/admin/scraper/run
- * Trigger a manual scrape
- *
- * Request body (optional):
- *   - dryRun: boolean - Skip database writes for safe testing
- *   - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
- */
-router.post('/scraper/run', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-    const { dryRun = false, htmlContent = null } = req.body || {};
-
-    // Check if scraper is already running
-    if (isScraperRunning()) {
-      return res.status(409).json({ error: 'Scrape already in progress' });
-    }
-
-    // Generate job ID and acquire lock
-    const jobId = crypto.randomUUID();
-    acquireLock(jobId);
-
-    // Log admin action
-    await logActivity(db, {
-      userId: req.user._id.toString(),
-      action: 'ADMIN_TRIGGER_SCRAPE',
-      metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
-    });
-
-    // Start scrape asynchronously (do not await - return 202 immediately)
-    runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
-      .catch((error) => {
-        console.error('Async scrape failed:', error.message);
-      })
-      .finally(() => releaseLock());
-
-    // Return immediately with job ID
-    res.status(202).json({
-      data: {
-        jobId,
-        status: 'started',
-        dryRun,
-        message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
-      }
-    });
-
-  } catch (error) {
-    console.error('Error triggering scrape:', error);
-    res.status(500).json({ error: 'Failed to start scrape job' });
-  }
-});
-
 /**
  * GET /api/admin/scraper/status
  * Get current scraper status including running state, last run details,