Add GET /api/admin/scraper/history endpoint with pagination

Implement the scraper history endpoint that returns past scrape run
records from the scraper_runs collection. Features include:

- GET /api/admin/scraper/history protected by requireAuth + requireAdmin
- Pagination via limit (1-100, default 30) and offset (default 0) params
- Input validation returning 400 for invalid limit/offset values
- Results sorted by startedAt descending (newest first)
- Graceful error handling returning 503 on database failures
- 20 new tests covering auth, pagination, validation, and error cases

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

__tests__/scraper/scraperRoutes.test.js

@@ -18,6 +18,18 @@
  *   - lastRun details from database
  *   - nextScheduledRun and schedule fields
  *   - 503 on database errors
+ * - GET /api/admin/scraper/history
+ *   - Authentication and authorization (401/403)
+ *   - Default pagination (limit 30, offset 0)
+ *   - Custom limit and offset query params
+ *   - Limit validation (< 1, NaN returns 400)
+ *   - Offset validation (< 0, NaN returns 400)
+ *   - Limit capped at 100
+ *   - Results sorted by startedAt descending
+ *   - Returns array of run records in data field
+ *   - Returns empty array when no runs exist
+ *   - Returns empty array when offset beyond total
+ *   - 503 on database errors
  */
 
 const request = require('supertest');
@@ -843,4 +855,482 @@ describe('Scraper Routes', () => {
       });
     });
   });
+
+  // ============================================================
+  // GET /api/admin/scraper/history
+  // ============================================================
+  describe('GET /api/admin/scraper/history', () => {
+
+    describe('Authentication and Authorization', () => {
+      it('should return 401 without authentication', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .expect(401);
+
+        expect(res.body).toHaveProperty('error');
+      });
+
+      it('should return 403 for non-admin user', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, regularUser);
+        const token = generateTestToken(regularUser._id);
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(403);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toBe('Admin access required');
+      });
+    });
+
+    describe('Successful history response (admin)', () => {
+      let admin;
+      let token;
+
+      beforeEach(async () => {
+        admin = createTestAdmin();
+        await insertTestUser(db, admin);
+        token = generateTestToken(admin._id);
+      });
+
+      it('should return 200 with data array for admin', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body).toHaveProperty('data');
+        expect(Array.isArray(res.body.data)).toBe(true);
+      });
+
+      it('should return empty array when no runs exist', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toEqual([]);
+      });
+
+      it('should return run records with correct fields', async () => {
+        await db.collection('scraper_runs').insertOne({
+          jobId: 'history-job-001',
+          trigger: 'scheduled',
+          status: 'success',
+          startedAt: '2026-02-05T06:00:00.000Z',
+          completedAt: '2026-02-05T06:00:12.345Z',
+          duration: 12345,
+          unitsProcessed: 50,
+          pricesInserted: 48,
+          newUnitsCount: 2,
+          rentedUnitsCount: 1,
+          staleUnitsCount: 0,
+          errors: [],
+          recordedAt: new Date('2026-02-05T06:00:12.345Z')
+        });
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(1);
+        const record = res.body.data[0];
+        expect(record.jobId).toBe('history-job-001');
+        expect(record.trigger).toBe('scheduled');
+        expect(record.status).toBe('success');
+        expect(record.startedAt).toBe('2026-02-05T06:00:00.000Z');
+        expect(record.completedAt).toBe('2026-02-05T06:00:12.345Z');
+        expect(record.duration).toBe(12345);
+        expect(record.unitsProcessed).toBe(50);
+        expect(record.pricesInserted).toBe(48);
+        expect(record.newUnitsCount).toBe(2);
+        expect(record.rentedUnitsCount).toBe(1);
+        expect(record.staleUnitsCount).toBe(0);
+        expect(record.errors).toEqual([]);
+      });
+
+      it('should sort results by startedAt descending (newest first)', async () => {
+        // Insert runs in non-chronological order
+        await db.collection('scraper_runs').insertMany([
+          {
+            jobId: 'oldest-job',
+            trigger: 'scheduled',
+            status: 'success',
+            startedAt: '2026-02-03T06:00:00.000Z',
+            completedAt: '2026-02-03T06:00:10.000Z',
+            duration: 10000,
+            unitsProcessed: 40,
+            pricesInserted: 40,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: new Date('2026-02-03T06:00:10.000Z')
+          },
+          {
+            jobId: 'newest-job',
+            trigger: 'manual',
+            status: 'success',
+            startedAt: '2026-02-05T14:00:00.000Z',
+            completedAt: '2026-02-05T14:00:08.000Z',
+            duration: 8000,
+            unitsProcessed: 52,
+            pricesInserted: 50,
+            newUnitsCount: 1,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: new Date('2026-02-05T14:00:08.000Z')
+          },
+          {
+            jobId: 'middle-job',
+            trigger: 'scheduled',
+            status: 'failed',
+            startedAt: '2026-02-04T06:00:00.000Z',
+            completedAt: '2026-02-04T06:00:32.000Z',
+            duration: 32000,
+            unitsProcessed: 0,
+            pricesInserted: 0,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: ['HTTP request failed'],
+            recordedAt: new Date('2026-02-04T06:00:32.000Z')
+          }
+        ]);
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(3);
+        expect(res.body.data[0].jobId).toBe('newest-job');
+        expect(res.body.data[1].jobId).toBe('middle-job');
+        expect(res.body.data[2].jobId).toBe('oldest-job');
+      });
+
+      it('should default limit to 30 records', async () => {
+        // Insert 35 records
+        const runs = [];
+        for (let i = 0; i < 35; i++) {
+          const date = new Date('2026-01-01T06:00:00.000Z');
+          date.setDate(date.getDate() + i);
+          runs.push({
+            jobId: `job-${String(i).padStart(3, '0')}`,
+            trigger: 'scheduled',
+            status: 'success',
+            startedAt: date.toISOString(),
+            completedAt: date.toISOString(),
+            duration: 10000,
+            unitsProcessed: 50,
+            pricesInserted: 50,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: date
+          });
+        }
+        await db.collection('scraper_runs').insertMany(runs);
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(30);
+      });
+
+      it('should respect custom limit query parameter', async () => {
+        // Insert 10 records
+        const runs = [];
+        for (let i = 0; i < 10; i++) {
+          const date = new Date('2026-01-01T06:00:00.000Z');
+          date.setDate(date.getDate() + i);
+          runs.push({
+            jobId: `job-limit-${i}`,
+            trigger: 'scheduled',
+            status: 'success',
+            startedAt: date.toISOString(),
+            completedAt: date.toISOString(),
+            duration: 10000,
+            unitsProcessed: 50,
+            pricesInserted: 50,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: date
+          });
+        }
+        await db.collection('scraper_runs').insertMany(runs);
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=5')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(5);
+      });
+
+      it('should cap limit at 100 even if higher value requested', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=200')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        // Should not error - just caps at 100
+        expect(res.body).toHaveProperty('data');
+        expect(Array.isArray(res.body.data)).toBe(true);
+      });
+
+      it('should apply offset to skip records', async () => {
+        // Insert 5 records in order
+        const runs = [];
+        for (let i = 0; i < 5; i++) {
+          const date = new Date('2026-02-01T06:00:00.000Z');
+          date.setDate(date.getDate() + i);
+          runs.push({
+            jobId: `offset-job-${i}`,
+            trigger: 'scheduled',
+            status: 'success',
+            startedAt: date.toISOString(),
+            completedAt: date.toISOString(),
+            duration: 10000,
+            unitsProcessed: 50,
+            pricesInserted: 50,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: date
+          });
+        }
+        await db.collection('scraper_runs').insertMany(runs);
+
+        // Skip first 2 (newest two), get remaining 3
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=2&limit=10')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(3);
+        // The 3rd newest should be first (offset skips the 2 newest)
+        expect(res.body.data[0].jobId).toBe('offset-job-2');
+      });
+
+      it('should return empty array when offset exceeds total records', async () => {
+        await db.collection('scraper_runs').insertOne({
+          jobId: 'single-job',
+          trigger: 'manual',
+          status: 'success',
+          startedAt: '2026-02-05T06:00:00.000Z',
+          completedAt: '2026-02-05T06:00:10.000Z',
+          duration: 10000,
+          unitsProcessed: 50,
+          pricesInserted: 50,
+          newUnitsCount: 0,
+          rentedUnitsCount: 0,
+          staleUnitsCount: 0,
+          errors: [],
+          recordedAt: new Date()
+        });
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=100')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toEqual([]);
+      });
+
+      it('should accept offset of 0 as valid', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=0')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body).toHaveProperty('data');
+        expect(Array.isArray(res.body.data)).toBe(true);
+      });
+
+      it('should accept limit of 1 as valid', async () => {
+        // Insert 3 records
+        const runs = [];
+        for (let i = 0; i < 3; i++) {
+          const date = new Date('2026-02-01T06:00:00.000Z');
+          date.setDate(date.getDate() + i);
+          runs.push({
+            jobId: `limit1-job-${i}`,
+            trigger: 'scheduled',
+            status: 'success',
+            startedAt: date.toISOString(),
+            completedAt: date.toISOString(),
+            duration: 10000,
+            unitsProcessed: 50,
+            pricesInserted: 50,
+            newUnitsCount: 0,
+            rentedUnitsCount: 0,
+            staleUnitsCount: 0,
+            errors: [],
+            recordedAt: date
+          });
+        }
+        await db.collection('scraper_runs').insertMany(runs);
+
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=1')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body.data).toHaveLength(1);
+      });
+
+      it('should accept limit of 100 as valid', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=100')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(200);
+
+        expect(res.body).toHaveProperty('data');
+        expect(Array.isArray(res.body.data)).toBe(true);
+      });
+    });
+
+    describe('Validation errors', () => {
+      let admin;
+      let token;
+
+      beforeEach(async () => {
+        admin = createTestAdmin();
+        await insertTestUser(db, admin);
+        token = generateTestToken(admin._id);
+      });
+
+      it('should return 400 for limit less than 1', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=0')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/limit/i);
+      });
+
+      it('should return 400 for negative limit', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=-5')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/limit/i);
+      });
+
+      it('should return 400 for non-numeric limit', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=abc')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/limit/i);
+      });
+
+      it('should return 400 for negative offset', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=-1')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/offset/i);
+      });
+
+      it('should return 400 for non-numeric offset', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=xyz')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toMatch(/offset/i);
+      });
+
+      it('should return specific error message for invalid limit', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?limit=0')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body.error).toBe('Limit must be between 1 and 100');
+      });
+
+      it('should return specific error message for invalid offset', async () => {
+        const res = await request(app)
+          .get('/api/admin/scraper/history?offset=-1')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(400);
+
+        expect(res.body.error).toBe('Invalid offset parameter');
+      });
+    });
+
+    describe('Error handling', () => {
+      let admin;
+      let token;
+
+      beforeEach(async () => {
+        admin = createTestAdmin();
+        await insertTestUser(db, admin);
+        token = generateTestToken(admin._id);
+      });
+
+      it('should return 503 on database error', async () => {
+        const express = require('express');
+        const cookieParser = require('cookie-parser');
+
+        const brokenApp = express();
+        brokenApp.use(express.json());
+        brokenApp.use(cookieParser());
+
+        // Proxy db: real db for users, broken for scraper_runs
+        const proxyDb = {
+          collection: jest.fn((name) => {
+            if (name === 'scraper_runs') {
+              return {
+                find: jest.fn().mockReturnValue({
+                  sort: jest.fn().mockReturnValue({
+                    skip: jest.fn().mockReturnValue({
+                      limit: jest.fn().mockReturnValue({
+                        toArray: jest.fn().mockRejectedValue(new Error('Database connection lost'))
+                      })
+                    })
+                  })
+                }),
+                findOne: jest.fn().mockRejectedValue(new Error('Database connection lost'))
+              };
+            }
+            return db.collection(name);
+          })
+        };
+        brokenApp.locals.db = proxyDb;
+
+        const adminRoutes = require('../../routes/admin');
+        brokenApp.use('/api/admin', adminRoutes);
+
+        const res = await request(brokenApp)
+          .get('/api/admin/scraper/history')
+          .set('Cookie', [`auth_token=${token}`])
+          .expect(503);
+
+        expect(res.body).toHaveProperty('error');
+        expect(res.body.error).toBe('Service temporarily unavailable');
+      });
+    });
+  });
 });

routes/admin.js

@@ -1250,5 +1250,55 @@ router.get('/scraper/status', async (req, res) => {
   }
 });
 
+/**
+ * GET /api/admin/scraper/history
+ * Get scraper run history with pagination
+ *
+ * Query params:
+ *   - limit: Number of records (1-100, default 30)
+ *   - offset: Number of records to skip (default 0)
+ */
+router.get('/scraper/history', async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+
+    // Parse and validate pagination parameters
+    let limit = parseInt(req.query.limit);
+    let offset = parseInt(req.query.offset);
+
+    // Validate limit
+    if (req.query.limit !== undefined) {
+      if (isNaN(limit) || limit < 1) {
+        return res.status(400).json({ error: 'Limit must be between 1 and 100' });
+      }
+      limit = Math.min(limit, 100);
+    } else {
+      limit = 30;
+    }
+
+    // Validate offset
+    if (req.query.offset !== undefined) {
+      if (isNaN(offset) || offset < 0) {
+        return res.status(400).json({ error: 'Invalid offset parameter' });
+      }
+    } else {
+      offset = 0;
+    }
+
+    const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
+      .find({})
+      .sort({ startedAt: -1 })
+      .skip(offset)
+      .limit(limit)
+      .toArray();
+
+    res.json({ data: history });
+
+  } catch (error) {
+    console.error('Error fetching scraper history:', error);
+    res.status(503).json({ error: 'Service temporarily unavailable' });
+  }
+});
+
 module.exports = router;
 module.exports.clearStatsCache = clearStatsCache;