Add GET /api/admin/scraper/history endpoint with pagination
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 42s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 42s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Implement the scraper history endpoint that returns past scrape run records from the scraper_runs collection. Features include: - GET /api/admin/scraper/history protected by requireAuth + requireAdmin - Pagination via limit (1-100, default 30) and offset (default 0) params - Input validation returning 400 for invalid limit/offset values - Results sorted by startedAt descending (newest first) - Graceful error handling returning 503 on database failures - 20 new tests covering auth, pagination, validation, and error cases
This commit is contained in:
@ -1250,5 +1250,55 @@ router.get('/scraper/status', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/admin/scraper/history
|
||||
* Get scraper run history with pagination
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Number of records (1-100, default 30)
|
||||
* - offset: Number of records to skip (default 0)
|
||||
*/
|
||||
router.get('/scraper/history', async (req, res) => {
|
||||
try {
|
||||
const db = req.app.locals.db;
|
||||
|
||||
// Parse and validate pagination parameters
|
||||
let limit = parseInt(req.query.limit);
|
||||
let offset = parseInt(req.query.offset);
|
||||
|
||||
// Validate limit
|
||||
if (req.query.limit !== undefined) {
|
||||
if (isNaN(limit) || limit < 1) {
|
||||
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
|
||||
}
|
||||
limit = Math.min(limit, 100);
|
||||
} else {
|
||||
limit = 30;
|
||||
}
|
||||
|
||||
// Validate offset
|
||||
if (req.query.offset !== undefined) {
|
||||
if (isNaN(offset) || offset < 0) {
|
||||
return res.status(400).json({ error: 'Invalid offset parameter' });
|
||||
}
|
||||
} else {
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
|
||||
.find({})
|
||||
.sort({ startedAt: -1 })
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
|
||||
res.json({ data: history });
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error fetching scraper history:', error);
|
||||
res.status(503).json({ error: 'Service temporarily unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
module.exports.clearStatsCache = clearStatsCache;
|
||||
|
||||
Reference in New Issue
Block a user