Add per-user rate limiting to scraper trigger endpoint
All checks were successful
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
All checks were successful
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
Implement rate limiting for POST /api/admin/scraper/run with a cap of 5 requests per hour per admin user. When exceeded, the endpoint returns 429 Too Many Requests with a Retry-After header indicating seconds until the window resets. Rate limit tracking uses an in-memory Map keyed by user ID. The check runs before the mutex check so rate-limited users get a 429 rather than a misleading 409 conflict. Scheduled/cron-triggered runs are not counted against any user's limit. Adds 6 new tests covering: allow up to 5 requests, reject 6th with 429, Retry-After header presence, per-user isolation, window expiry reset, and rate-limit-before-mutex ordering.
This commit is contained in:
@ -1523,6 +1523,190 @@ describe('Scraper Routes', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Rate Limiting for POST /api/admin/scraper/run (SCRAPE-22)
|
||||
// ============================================================
|
||||
describe('rate limiting', () => {
|
||||
let adminUser;
|
||||
let adminToken;
|
||||
|
||||
beforeEach(async () => {
|
||||
adminUser = createTestAdmin();
|
||||
await insertTestUser(db, adminUser);
|
||||
adminToken = generateTestToken(adminUser._id);
|
||||
|
||||
// Reset the rate limiter between tests
|
||||
const { resetRateLimiter } = require('../../routes/admin');
|
||||
resetRateLimiter();
|
||||
});
|
||||
|
||||
it('should allow the first 5 requests within the rate limit window', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
// Reset scraper state to idle before each request so mutex does not block
|
||||
scraperJob.__reset();
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(202);
|
||||
|
||||
// Wait for async scrape to release lock
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
});
|
||||
|
||||
it('should return 429 when the 6th request exceeds the rate limit', async () => {
|
||||
// Make 5 successful requests
|
||||
for (let i = 0; i < 5; i++) {
|
||||
scraperJob.__reset();
|
||||
|
||||
await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// 6th request should be rate limited
|
||||
scraperJob.__reset();
|
||||
const res = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(429);
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/rate limit/i);
|
||||
});
|
||||
|
||||
it('should include Retry-After header in 429 response', async () => {
|
||||
// Make 5 successful requests
|
||||
for (let i = 0; i < 5; i++) {
|
||||
scraperJob.__reset();
|
||||
|
||||
await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// 6th request
|
||||
scraperJob.__reset();
|
||||
const res = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(429);
|
||||
expect(res.headers).toHaveProperty('retry-after');
|
||||
|
||||
const retryAfter = parseInt(res.headers['retry-after']);
|
||||
expect(retryAfter).toBeGreaterThan(0);
|
||||
// Should be less than or equal to 3600 seconds (1 hour)
|
||||
expect(retryAfter).toBeLessThanOrEqual(3600);
|
||||
});
|
||||
|
||||
it('should track rate limits per user (different admins have separate limits)', async () => {
|
||||
// Create a second admin
|
||||
const admin2 = createTestAdmin();
|
||||
await insertTestUser(db, admin2);
|
||||
const admin2Token = generateTestToken(admin2._id);
|
||||
|
||||
// Use up first admin's rate limit (5 requests)
|
||||
for (let i = 0; i < 5; i++) {
|
||||
scraperJob.__reset();
|
||||
|
||||
await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// Verify first admin is rate limited
|
||||
scraperJob.__reset();
|
||||
const res1 = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
expect(res1.status).toBe(429);
|
||||
|
||||
// Second admin should still be able to make requests
|
||||
scraperJob.__reset();
|
||||
const res2 = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${admin2Token}`])
|
||||
.send({});
|
||||
expect(res2.status).toBe(202);
|
||||
});
|
||||
|
||||
it('should reset rate limit after window expires', async () => {
|
||||
// Access the rate limiter internals for testing
|
||||
const adminModule = require('../../routes/admin');
|
||||
|
||||
// Use up the rate limit (5 requests)
|
||||
for (let i = 0; i < 5; i++) {
|
||||
scraperJob.__reset();
|
||||
|
||||
await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// Verify rate limited
|
||||
scraperJob.__reset();
|
||||
const blockedRes = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
expect(blockedRes.status).toBe(429);
|
||||
|
||||
// Simulate window expiry by resetting the rate limiter
|
||||
adminModule.resetRateLimiter();
|
||||
|
||||
// Should be allowed again after window reset
|
||||
scraperJob.__reset();
|
||||
const allowedRes = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
expect(allowedRes.status).toBe(202);
|
||||
});
|
||||
|
||||
it('should check rate limit before mutex (rate limit takes precedence)', async () => {
|
||||
// Use up rate limit
|
||||
for (let i = 0; i < 5; i++) {
|
||||
scraperJob.__reset();
|
||||
|
||||
await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
// Set scraper as running (would normally get 409)
|
||||
scraperJob._setState(true, 'existing-job');
|
||||
|
||||
// Should get 429 (rate limit), not 409 (mutex conflict)
|
||||
const res = await request(app)
|
||||
.post('/api/admin/scraper/run')
|
||||
.set('Cookie', [`auth_token=${adminToken}`])
|
||||
.send({});
|
||||
expect(res.status).toBe(429);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Middleware ordering: requireAuth runs before requireAdmin
|
||||
// ============================================================
|
||||
|
||||
Reference in New Issue
Block a user