__tests__/scraper/scraperRoutes.test.js

@@ -1523,6 +1523,190 @@ describe('Scraper Routes', () => {
       });
     });
 
+    // ============================================================
+    // Rate Limiting for POST /api/admin/scraper/run (SCRAPE-22)
+    // ============================================================
+    describe('rate limiting', () => {
+      let adminUser;
+      let adminToken;
+
+      beforeEach(async () => {
+        adminUser = createTestAdmin();
+        await insertTestUser(db, adminUser);
+        adminToken = generateTestToken(adminUser._id);
+
+        // Reset the rate limiter between tests
+        const { resetRateLimiter } = require('../../routes/admin');
+        resetRateLimiter();
+      });
+
+      it('should allow the first 5 requests within the rate limit window', async () => {
+        for (let i = 0; i < 5; i++) {
+          // Reset scraper state to idle before each request so mutex does not block
+          scraperJob.__reset();
+
+          const res = await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          expect(res.status).toBe(202);
+
+          // Wait for async scrape to release lock
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+      });
+
+      it('should return 429 when the 6th request exceeds the rate limit', async () => {
+        // Make 5 successful requests
+        for (let i = 0; i < 5; i++) {
+          scraperJob.__reset();
+
+          await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+
+        // 6th request should be rate limited
+        scraperJob.__reset();
+        const res = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+
+        expect(res.status).toBe(429);
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/rate limit/i);
+      });
+
+      it('should include Retry-After header in 429 response', async () => {
+        // Make 5 successful requests
+        for (let i = 0; i < 5; i++) {
+          scraperJob.__reset();
+
+          await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+
+        // 6th request
+        scraperJob.__reset();
+        const res = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+
+        expect(res.status).toBe(429);
+        expect(res.headers).toHaveProperty('retry-after');
+
+        const retryAfter = parseInt(res.headers['retry-after']);
+        expect(retryAfter).toBeGreaterThan(0);
+        // Should be less than or equal to 3600 seconds (1 hour)
+        expect(retryAfter).toBeLessThanOrEqual(3600);
+      });
+
+      it('should track rate limits per user (different admins have separate limits)', async () => {
+        // Create a second admin
+        const admin2 = createTestAdmin();
+        await insertTestUser(db, admin2);
+        const admin2Token = generateTestToken(admin2._id);
+
+        // Use up first admin's rate limit (5 requests)
+        for (let i = 0; i < 5; i++) {
+          scraperJob.__reset();
+
+          await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+
+        // Verify first admin is rate limited
+        scraperJob.__reset();
+        const res1 = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+        expect(res1.status).toBe(429);
+
+        // Second admin should still be able to make requests
+        scraperJob.__reset();
+        const res2 = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${admin2Token}`])
+          .send({});
+        expect(res2.status).toBe(202);
+      });
+
+      it('should reset rate limit after window expires', async () => {
+        // Access the rate limiter internals for testing
+        const adminModule = require('../../routes/admin');
+
+        // Use up the rate limit (5 requests)
+        for (let i = 0; i < 5; i++) {
+          scraperJob.__reset();
+
+          await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+
+        // Verify rate limited
+        scraperJob.__reset();
+        const blockedRes = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+        expect(blockedRes.status).toBe(429);
+
+        // Simulate window expiry by resetting the rate limiter
+        adminModule.resetRateLimiter();
+
+        // Should be allowed again after window reset
+        scraperJob.__reset();
+        const allowedRes = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+        expect(allowedRes.status).toBe(202);
+      });
+
+      it('should check rate limit before mutex (rate limit takes precedence)', async () => {
+        // Use up rate limit
+        for (let i = 0; i < 5; i++) {
+          scraperJob.__reset();
+
+          await request(app)
+            .post('/api/admin/scraper/run')
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({});
+
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+
+        // Set scraper as running (would normally get 409)
+        scraperJob._setState(true, 'existing-job');
+
+        // Should get 429 (rate limit), not 409 (mutex conflict)
+        const res = await request(app)
+          .post('/api/admin/scraper/run')
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+        expect(res.status).toBe(429);
+      });
+    });
+
     // ============================================================
     // Middleware ordering: requireAuth runs before requireAdmin
     // ============================================================

routes/admin.js

@@ -1162,6 +1162,53 @@ const scraperConfig = require('../config/scraper');
 // Logger for scraper admin routes
 const scraperRouteLogger = createLogger('scraper-admin');
 
+// Rate limiting for manual scrape trigger (per-user, in-memory)
+const RATE_LIMIT_MAX_REQUESTS = 5;
+const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
+const rateLimitStore = new Map();
+
+/**
+ * Check rate limit for a given user ID.
+ * Returns an object indicating whether the request is allowed.
+ *
+ * @param {string} userId - The user ID to check
+ * @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
+ */
+function checkRateLimit(userId) {
+  const now = Date.now();
+  const userKey = userId.toString();
+
+  if (!rateLimitStore.has(userKey)) {
+    rateLimitStore.set(userKey, []);
+  }
+
+  const timestamps = rateLimitStore.get(userKey);
+
+  // Remove timestamps outside the current window
+  const windowStart = now - RATE_LIMIT_WINDOW_MS;
+  const validTimestamps = timestamps.filter(ts => ts > windowStart);
+  rateLimitStore.set(userKey, validTimestamps);
+
+  if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
+    // Calculate when the oldest request in the window will expire
+    const oldestTimestamp = validTimestamps[0];
+    const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
+    const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
+    return { allowed: false, retryAfterSeconds };
+  }
+
+  // Record this request
+  validTimestamps.push(now);
+  return { allowed: true, retryAfterSeconds: null };
+}
+
+/**
+ * Reset the rate limiter (for testing)
+ */
+function resetRateLimiter() {
+  rateLimitStore.clear();
+}
+
 /**
  * POST /api/admin/scraper/run
  * Trigger a manual scrape
@@ -1175,6 +1222,15 @@ router.post('/scraper/run', async (req, res) => {
     const db = req.app.locals.db;
     const { dryRun = false, htmlContent = null } = req.body || {};
 
+    // Check rate limit (per-user, before mutex check)
+    const rateLimitResult = checkRateLimit(req.user._id);
+    if (!rateLimitResult.allowed) {
+      res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
+      return res.status(429).json({
+        error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
+      });
+    }
+
     // Check if scraper is already running
     if (isScraperRunning()) {
       return res.status(409).json({ error: 'Scrape already in progress' });
@@ -1318,3 +1374,4 @@ router.get('/scraper/history', async (req, res) => {
 
 module.exports = router;
 module.exports.clearStatsCache = clearStatsCache;
+module.exports.resetRateLimiter = resetRateLimiter;