SCRAPE-18: Implement GET /admin/scraper/history endpoint #23
@ -18,6 +18,18 @@
|
||||
* - lastRun details from database
|
||||
* - nextScheduledRun and schedule fields
|
||||
* - 503 on database errors
|
||||
* - GET /api/admin/scraper/history
|
||||
* - Authentication and authorization (401/403)
|
||||
* - Default pagination (limit 30, offset 0)
|
||||
* - Custom limit and offset query params
|
||||
* - Limit validation (< 1, NaN returns 400)
|
||||
* - Offset validation (< 0, NaN returns 400)
|
||||
* - Limit capped at 100
|
||||
* - Results sorted by startedAt descending
|
||||
* - Returns array of run records in data field
|
||||
* - Returns empty array when no runs exist
|
||||
* - Returns empty array when offset beyond total
|
||||
* - 503 on database errors
|
||||
*/
|
||||
|
||||
const request = require('supertest');
|
||||
@ -843,4 +855,482 @@ describe('Scraper Routes', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// GET /api/admin/scraper/history
|
||||
// ============================================================
|
||||
describe('GET /api/admin/scraper/history', () => {
|
||||
|
||||
describe('Authentication and Authorization', () => {
|
||||
it('should return 401 without authentication', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.expect(401);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
});
|
||||
|
||||
it('should return 403 for non-admin user', async () => {
|
||||
const regularUser = createTestUser({ role: 'user' });
|
||||
await insertTestUser(db, regularUser);
|
||||
const token = generateTestToken(regularUser._id);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(403);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toBe('Admin access required');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Successful history response (admin)', () => {
|
||||
let admin;
|
||||
let token;
|
||||
|
||||
beforeEach(async () => {
|
||||
admin = createTestAdmin();
|
||||
await insertTestUser(db, admin);
|
||||
token = generateTestToken(admin._id);
|
||||
});
|
||||
|
||||
it('should return 200 with data array for admin', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body).toHaveProperty('data');
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return empty array when no runs exist', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return run records with correct fields', async () => {
|
||||
await db.collection('scraper_runs').insertOne({
|
||||
jobId: 'history-job-001',
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: '2026-02-05T06:00:00.000Z',
|
||||
completedAt: '2026-02-05T06:00:12.345Z',
|
||||
duration: 12345,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 48,
|
||||
newUnitsCount: 2,
|
||||
rentedUnitsCount: 1,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: new Date('2026-02-05T06:00:12.345Z')
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(1);
|
||||
const record = res.body.data[0];
|
||||
expect(record.jobId).toBe('history-job-001');
|
||||
expect(record.trigger).toBe('scheduled');
|
||||
expect(record.status).toBe('success');
|
||||
expect(record.startedAt).toBe('2026-02-05T06:00:00.000Z');
|
||||
expect(record.completedAt).toBe('2026-02-05T06:00:12.345Z');
|
||||
expect(record.duration).toBe(12345);
|
||||
expect(record.unitsProcessed).toBe(50);
|
||||
expect(record.pricesInserted).toBe(48);
|
||||
expect(record.newUnitsCount).toBe(2);
|
||||
expect(record.rentedUnitsCount).toBe(1);
|
||||
expect(record.staleUnitsCount).toBe(0);
|
||||
expect(record.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should sort results by startedAt descending (newest first)', async () => {
|
||||
// Insert runs in non-chronological order
|
||||
await db.collection('scraper_runs').insertMany([
|
||||
{
|
||||
jobId: 'oldest-job',
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: '2026-02-03T06:00:00.000Z',
|
||||
completedAt: '2026-02-03T06:00:10.000Z',
|
||||
duration: 10000,
|
||||
unitsProcessed: 40,
|
||||
pricesInserted: 40,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: new Date('2026-02-03T06:00:10.000Z')
|
||||
},
|
||||
{
|
||||
jobId: 'newest-job',
|
||||
trigger: 'manual',
|
||||
status: 'success',
|
||||
startedAt: '2026-02-05T14:00:00.000Z',
|
||||
completedAt: '2026-02-05T14:00:08.000Z',
|
||||
duration: 8000,
|
||||
unitsProcessed: 52,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 1,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: new Date('2026-02-05T14:00:08.000Z')
|
||||
},
|
||||
{
|
||||
jobId: 'middle-job',
|
||||
trigger: 'scheduled',
|
||||
status: 'failed',
|
||||
startedAt: '2026-02-04T06:00:00.000Z',
|
||||
completedAt: '2026-02-04T06:00:32.000Z',
|
||||
duration: 32000,
|
||||
unitsProcessed: 0,
|
||||
pricesInserted: 0,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: ['HTTP request failed'],
|
||||
recordedAt: new Date('2026-02-04T06:00:32.000Z')
|
||||
}
|
||||
]);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(3);
|
||||
expect(res.body.data[0].jobId).toBe('newest-job');
|
||||
expect(res.body.data[1].jobId).toBe('middle-job');
|
||||
expect(res.body.data[2].jobId).toBe('oldest-job');
|
||||
});
|
||||
|
||||
it('should default limit to 30 records', async () => {
|
||||
// Insert 35 records
|
||||
const runs = [];
|
||||
for (let i = 0; i < 35; i++) {
|
||||
const date = new Date('2026-01-01T06:00:00.000Z');
|
||||
date.setDate(date.getDate() + i);
|
||||
runs.push({
|
||||
jobId: `job-${String(i).padStart(3, '0')}`,
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: date.toISOString(),
|
||||
completedAt: date.toISOString(),
|
||||
duration: 10000,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: date
|
||||
});
|
||||
}
|
||||
await db.collection('scraper_runs').insertMany(runs);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(30);
|
||||
});
|
||||
|
||||
it('should respect custom limit query parameter', async () => {
|
||||
// Insert 10 records
|
||||
const runs = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const date = new Date('2026-01-01T06:00:00.000Z');
|
||||
date.setDate(date.getDate() + i);
|
||||
runs.push({
|
||||
jobId: `job-limit-${i}`,
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: date.toISOString(),
|
||||
completedAt: date.toISOString(),
|
||||
duration: 10000,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: date
|
||||
});
|
||||
}
|
||||
await db.collection('scraper_runs').insertMany(runs);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=5')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('should cap limit at 100 even if higher value requested', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=200')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
// Should not error - just caps at 100
|
||||
expect(res.body).toHaveProperty('data');
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
|
||||
it('should apply offset to skip records', async () => {
|
||||
// Insert 5 records in order
|
||||
const runs = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const date = new Date('2026-02-01T06:00:00.000Z');
|
||||
date.setDate(date.getDate() + i);
|
||||
runs.push({
|
||||
jobId: `offset-job-${i}`,
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: date.toISOString(),
|
||||
completedAt: date.toISOString(),
|
||||
duration: 10000,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: date
|
||||
});
|
||||
}
|
||||
await db.collection('scraper_runs').insertMany(runs);
|
||||
|
||||
// Skip first 2 (newest two), get remaining 3
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=2&limit=10')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(3);
|
||||
// The 3rd newest should be first (offset skips the 2 newest)
|
||||
expect(res.body.data[0].jobId).toBe('offset-job-2');
|
||||
});
|
||||
|
||||
it('should return empty array when offset exceeds total records', async () => {
|
||||
await db.collection('scraper_runs').insertOne({
|
||||
jobId: 'single-job',
|
||||
trigger: 'manual',
|
||||
status: 'success',
|
||||
startedAt: '2026-02-05T06:00:00.000Z',
|
||||
completedAt: '2026-02-05T06:00:10.000Z',
|
||||
duration: 10000,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: new Date()
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=100')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toEqual([]);
|
||||
});
|
||||
|
||||
it('should accept offset of 0 as valid', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=0')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body).toHaveProperty('data');
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
|
||||
it('should accept limit of 1 as valid', async () => {
|
||||
// Insert 3 records
|
||||
const runs = [];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const date = new Date('2026-02-01T06:00:00.000Z');
|
||||
date.setDate(date.getDate() + i);
|
||||
runs.push({
|
||||
jobId: `limit1-job-${i}`,
|
||||
trigger: 'scheduled',
|
||||
status: 'success',
|
||||
startedAt: date.toISOString(),
|
||||
completedAt: date.toISOString(),
|
||||
duration: 10000,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 50,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: [],
|
||||
recordedAt: date
|
||||
});
|
||||
}
|
||||
await db.collection('scraper_runs').insertMany(runs);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=1')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should accept limit of 100 as valid', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=100')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body).toHaveProperty('data');
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Validation errors', () => {
|
||||
let admin;
|
||||
let token;
|
||||
|
||||
beforeEach(async () => {
|
||||
admin = createTestAdmin();
|
||||
await insertTestUser(db, admin);
|
||||
token = generateTestToken(admin._id);
|
||||
});
|
||||
|
||||
it('should return 400 for limit less than 1', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=0')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/limit/i);
|
||||
});
|
||||
|
||||
it('should return 400 for negative limit', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=-5')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/limit/i);
|
||||
});
|
||||
|
||||
it('should return 400 for non-numeric limit', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=abc')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/limit/i);
|
||||
});
|
||||
|
||||
it('should return 400 for negative offset', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=-1')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/offset/i);
|
||||
});
|
||||
|
||||
it('should return 400 for non-numeric offset', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=xyz')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toMatch(/offset/i);
|
||||
});
|
||||
|
||||
it('should return specific error message for invalid limit', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?limit=0')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body.error).toBe('Limit must be between 1 and 100');
|
||||
});
|
||||
|
||||
it('should return specific error message for invalid offset', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/scraper/history?offset=-1')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(400);
|
||||
|
||||
expect(res.body.error).toBe('Invalid offset parameter');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error handling', () => {
|
||||
let admin;
|
||||
let token;
|
||||
|
||||
beforeEach(async () => {
|
||||
admin = createTestAdmin();
|
||||
await insertTestUser(db, admin);
|
||||
token = generateTestToken(admin._id);
|
||||
});
|
||||
|
||||
it('should return 503 on database error', async () => {
|
||||
const express = require('express');
|
||||
const cookieParser = require('cookie-parser');
|
||||
|
||||
const brokenApp = express();
|
||||
brokenApp.use(express.json());
|
||||
brokenApp.use(cookieParser());
|
||||
|
||||
// Proxy db: real db for users, broken for scraper_runs
|
||||
const proxyDb = {
|
||||
collection: jest.fn((name) => {
|
||||
if (name === 'scraper_runs') {
|
||||
return {
|
||||
find: jest.fn().mockReturnValue({
|
||||
sort: jest.fn().mockReturnValue({
|
||||
skip: jest.fn().mockReturnValue({
|
||||
limit: jest.fn().mockReturnValue({
|
||||
toArray: jest.fn().mockRejectedValue(new Error('Database connection lost'))
|
||||
})
|
||||
})
|
||||
})
|
||||
}),
|
||||
findOne: jest.fn().mockRejectedValue(new Error('Database connection lost'))
|
||||
};
|
||||
}
|
||||
return db.collection(name);
|
||||
})
|
||||
};
|
||||
brokenApp.locals.db = proxyDb;
|
||||
|
||||
const adminRoutes = require('../../routes/admin');
|
||||
brokenApp.use('/api/admin', adminRoutes);
|
||||
|
||||
const res = await request(brokenApp)
|
||||
.get('/api/admin/scraper/history')
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(503);
|
||||
|
||||
expect(res.body).toHaveProperty('error');
|
||||
expect(res.body.error).toBe('Service temporarily unavailable');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1250,5 +1250,55 @@ router.get('/scraper/status', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/admin/scraper/history
|
||||
* Get scraper run history with pagination
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Number of records (1-100, default 30)
|
||||
* - offset: Number of records to skip (default 0)
|
||||
*/
|
||||
router.get('/scraper/history', async (req, res) => {
|
||||
try {
|
||||
const db = req.app.locals.db;
|
||||
|
||||
// Parse and validate pagination parameters
|
||||
let limit = parseInt(req.query.limit);
|
||||
let offset = parseInt(req.query.offset);
|
||||
|
||||
// Validate limit
|
||||
if (req.query.limit !== undefined) {
|
||||
if (isNaN(limit) || limit < 1) {
|
||||
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
|
||||
}
|
||||
limit = Math.min(limit, 100);
|
||||
} else {
|
||||
limit = 30;
|
||||
}
|
||||
|
||||
// Validate offset
|
||||
if (req.query.offset !== undefined) {
|
||||
if (isNaN(offset) || offset < 0) {
|
||||
return res.status(400).json({ error: 'Invalid offset parameter' });
|
||||
}
|
||||
} else {
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
|
||||
.find({})
|
||||
.sort({ startedAt: -1 })
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
|
||||
res.json({ data: history });
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error fetching scraper history:', error);
|
||||
res.status(503).json({ error: 'Service temporarily unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
module.exports.clearStatsCache = clearStatsCache;
|
||||
|
||||
Reference in New Issue
Block a user