SCRAPE-8: Implement markStaleUnits() (#13)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
@ -493,12 +493,55 @@ async function insertPrices(db, units, date, logger) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark units not in current scrape as stale/unavailable.
|
||||
* Uses updateMany to set available: false and marked_stale_date
|
||||
* for all units whose unit_code is NOT in the current scrape
|
||||
* and that are currently available.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Set<string>} currentUnitCodes - Unit codes from current scrape
|
||||
* @param {string} date - Current date in YYYY-MM-DD format
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object>} Update result
|
||||
*/
|
||||
async function markStaleUnits(db, currentUnitCodes, date, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.UNITS);
|
||||
|
||||
try {
|
||||
const result = await collection.updateMany(
|
||||
{
|
||||
unit_code: { $nin: [...currentUnitCodes] },
|
||||
available: true
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
available: false,
|
||||
marked_stale_date: date
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
logger.info('Stale units marked', { count: result.modifiedCount });
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to mark stale units', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchPage,
|
||||
parseUnits,
|
||||
convertDataTypes,
|
||||
upsertUnits,
|
||||
insertPrices,
|
||||
markStaleUnits,
|
||||
// Export helpers for testing
|
||||
parseInteger,
|
||||
parsePositiveInteger,
|
||||
|
||||
Reference in New Issue
Block a user