SCRAPE-10: Implement recordScraperRun() for scraper_runs (#15)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
2026-02-06 02:00:19 -07:00
committed by stephen
parent b4978caf31
commit d1f717891a
3 changed files with 364 additions and 16 deletions

View File

@ -619,6 +619,34 @@ async function updateDailySummary(db, summaryData, logger) {
}
}
/**
* Record scraper run to history collection.
* Called in the finally block of runScrape() to persist run metadata.
* This function must NOT throw errors - history recording should never break the scraper.
*
* @param {Db} db - MongoDB database instance
* @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
* @param {Object} logger - Logger instance
* @returns {Promise<Object|null>} Insert result, or null on failure
*/
async function recordScraperRun(db, runData, logger) {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
try {
const result = await collection.insertOne({
...runData,
recordedAt: new Date()
});
return result;
} catch (error) {
// Log but don't throw - recording history should not break scraper
logger.error('Failed to record scraper run', { errorMessage: error.message });
return null;
}
}
module.exports = {
fetchPage,
parseUnits,
@ -627,6 +655,7 @@ module.exports = {
insertPrices,
markStaleUnits,
updateDailySummary,
recordScraperRun,
// Export helpers for testing
getYesterday,
parseInteger,