SCRAPE-10: Implement recordScraperRun() for scraper_runs (#15)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
@ -619,6 +619,34 @@ async function updateDailySummary(db, summaryData, logger) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record scraper run to history collection.
|
||||
* Called in the finally block of runScrape() to persist run metadata.
|
||||
* This function must NOT throw errors - history recording should never break the scraper.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object|null>} Insert result, or null on failure
|
||||
*/
|
||||
async function recordScraperRun(db, runData, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
|
||||
|
||||
try {
|
||||
const result = await collection.insertOne({
|
||||
...runData,
|
||||
recordedAt: new Date()
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
// Log but don't throw - recording history should not break scraper
|
||||
logger.error('Failed to record scraper run', { errorMessage: error.message });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchPage,
|
||||
parseUnits,
|
||||
@ -627,6 +655,7 @@ module.exports = {
|
||||
insertPrices,
|
||||
markStaleUnits,
|
||||
updateDailySummary,
|
||||
recordScraperRun,
|
||||
// Export helpers for testing
|
||||
getYesterday,
|
||||
parseInteger,
|
||||
|
||||
Reference in New Issue
Block a user