SCRAPE-31: Add scraper indexes and validation collections (#30)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

services/scraperService.js

@@ -730,6 +730,37 @@ async function recordScraperRun(db, runData, logger) {
   }
 }
 
+// ============================================================
+// Index Management
+// ============================================================
+
+/**
+ * Create indexes on the scraper_runs collection.
+ * Should be called once during application startup.
+ * Idempotent - safe to call multiple times.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<void>}
+ */
+async function createScraperIndexes(db, logger) {
+  const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
+
+  // Compound index for querying runs by status sorted by most recent
+  await collection.createIndex(
+    { status: 1, startedAt: -1 },
+    { name: 'status_startedAt' }
+  );
+
+  // Index for sorting all runs by start time (most recent first)
+  await collection.createIndex(
+    { startedAt: -1 },
+    { name: 'startedAt_desc' }
+  );
+
+  logger.info('Scraper indexes created successfully');
+}
+
 // ============================================================
 // Main Orchestration Function
 // ============================================================
@@ -871,6 +902,7 @@ module.exports = {
   markStaleUnits,
   updateDailySummary,
   recordScraperRun,
+  createScraperIndexes,
   // Export helpers for testing
   getTodayUTC,
   getYesterdayUnitCodes,