Add GET /api/admin/scraper/history endpoint with pagination
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 41s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped

Implement the scraper history endpoint that returns past scrape run
records from the scraper_runs collection. Features include:

- GET /api/admin/scraper/history protected by requireAuth + requireAdmin
- Pagination via limit (1-100, default 30) and offset (default 0) params
- Input validation returning 400 for invalid limit/offset values
- Results sorted by startedAt descending (newest first)
- Graceful error handling returning 503 on database failures
- 20 new tests covering auth, pagination, validation, and error cases

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-06 22:27:09 -07:00
parent 9d0b9debbb
commit ea6928f933
2 changed files with 540 additions and 0 deletions

View File

@ -1250,5 +1250,55 @@ router.get('/scraper/status', async (req, res) => {
}
});
/**
* GET /api/admin/scraper/history
* Get scraper run history with pagination
*
* Query params:
* - limit: Number of records (1-100, default 30)
* - offset: Number of records to skip (default 0)
*/
router.get('/scraper/history', async (req, res) => {
try {
const db = req.app.locals.db;
// Parse and validate pagination parameters
let limit = parseInt(req.query.limit);
let offset = parseInt(req.query.offset);
// Validate limit
if (req.query.limit !== undefined) {
if (isNaN(limit) || limit < 1) {
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
}
limit = Math.min(limit, 100);
} else {
limit = 30;
}
// Validate offset
if (req.query.offset !== undefined) {
if (isNaN(offset) || offset < 0) {
return res.status(400).json({ error: 'Invalid offset parameter' });
}
} else {
offset = 0;
}
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.find({})
.sort({ startedAt: -1 })
.skip(offset)
.limit(limit)
.toArray();
res.json({ data: history });
} catch (error) {
console.error('Error fetching scraper history:', error);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
module.exports = router;
module.exports.clearStatsCache = clearStatsCache;