Add per-user rate limiting to scraper trigger endpoint
All checks were successful
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s

Implement rate limiting for POST /api/admin/scraper/run with a cap of
5 requests per hour per admin user. When exceeded, the endpoint returns
429 Too Many Requests with a Retry-After header indicating seconds
until the window resets.

Rate limit tracking uses an in-memory Map keyed by user ID. The check
runs before the mutex check so rate-limited users get a 429 rather
than a misleading 409 conflict. Scheduled/cron-triggered runs are not
counted against any user's limit.

Adds 6 new tests covering: allow up to 5 requests, reject 6th with
429, Retry-After header presence, per-user isolation, window expiry
reset, and rate-limit-before-mutex ordering.
This commit is contained in:
2026-02-06 23:15:31 -07:00
parent a1ee25ef17
commit a05c844b93
2 changed files with 241 additions and 0 deletions

View File

@ -1523,6 +1523,190 @@ describe('Scraper Routes', () => {
});
});
// ============================================================
// Rate Limiting for POST /api/admin/scraper/run (SCRAPE-22)
// ============================================================
describe('rate limiting', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
// Reset the rate limiter between tests
const { resetRateLimiter } = require('../../routes/admin');
resetRateLimiter();
});
it('should allow the first 5 requests within the rate limit window', async () => {
for (let i = 0; i < 5; i++) {
// Reset scraper state to idle before each request so mutex does not block
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// Wait for async scrape to release lock
await new Promise(resolve => setTimeout(resolve, 50));
}
});
it('should return 429 when the 6th request exceeds the rate limit', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request should be rate limited
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/rate limit/i);
});
it('should include Retry-After header in 429 response', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.headers).toHaveProperty('retry-after');
const retryAfter = parseInt(res.headers['retry-after']);
expect(retryAfter).toBeGreaterThan(0);
// Should be less than or equal to 3600 seconds (1 hour)
expect(retryAfter).toBeLessThanOrEqual(3600);
});
it('should track rate limits per user (different admins have separate limits)', async () => {
// Create a second admin
const admin2 = createTestAdmin();
await insertTestUser(db, admin2);
const admin2Token = generateTestToken(admin2._id);
// Use up first admin's rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify first admin is rate limited
scraperJob.__reset();
const res1 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res1.status).toBe(429);
// Second admin should still be able to make requests
scraperJob.__reset();
const res2 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${admin2Token}`])
.send({});
expect(res2.status).toBe(202);
});
it('should reset rate limit after window expires', async () => {
// Access the rate limiter internals for testing
const adminModule = require('../../routes/admin');
// Use up the rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify rate limited
scraperJob.__reset();
const blockedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(blockedRes.status).toBe(429);
// Simulate window expiry by resetting the rate limiter
adminModule.resetRateLimiter();
// Should be allowed again after window reset
scraperJob.__reset();
const allowedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(allowedRes.status).toBe(202);
});
it('should check rate limit before mutex (rate limit takes precedence)', async () => {
// Use up rate limit
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Set scraper as running (would normally get 409)
scraperJob._setState(true, 'existing-job');
// Should get 429 (rate limit), not 409 (mutex conflict)
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
});
});
// ============================================================
// Middleware ordering: requireAuth runs before requireAdmin
// ============================================================