SCRAPE-18: Implement GET /admin/scraper/history endpoint (#23)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
2026-02-06 22:34:46 -07:00
committed by stephen
parent 9d0b9debbb
commit d0538334a5
2 changed files with 540 additions and 0 deletions

View File

@ -1250,5 +1250,55 @@ router.get('/scraper/status', async (req, res) => {
}
});
/**
* GET /api/admin/scraper/history
* Get scraper run history with pagination
*
* Query params:
* - limit: Number of records (1-100, default 30)
* - offset: Number of records to skip (default 0)
*/
router.get('/scraper/history', async (req, res) => {
try {
const db = req.app.locals.db;
// Parse and validate pagination parameters
let limit = parseInt(req.query.limit);
let offset = parseInt(req.query.offset);
// Validate limit
if (req.query.limit !== undefined) {
if (isNaN(limit) || limit < 1) {
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
}
limit = Math.min(limit, 100);
} else {
limit = 30;
}
// Validate offset
if (req.query.offset !== undefined) {
if (isNaN(offset) || offset < 0) {
return res.status(400).json({ error: 'Invalid offset parameter' });
}
} else {
offset = 0;
}
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.find({})
.sort({ startedAt: -1 })
.skip(offset)
.limit(limit)
.toArray();
res.json({ data: history });
} catch (error) {
console.error('Error fetching scraper history:', error);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
module.exports = router;
module.exports.clearStatsCache = clearStatsCache;