Merge dev: Node.js scraper migration + CI fix #32

Merged
stephen merged 32 commits from dev into main 2026-02-08 20:35:29 -07:00
23 changed files with 10263 additions and 25 deletions
Showing only changes of commit 9d0b9debbb - Show all commits

File diff suppressed because it is too large Load Diff

View File

@ -1150,9 +1150,13 @@ const crypto = require('crypto');
const {
isScraperRunning,
acquireLock,
releaseLock
releaseLock,
getCurrentJobId,
getScheduleExpression,
getNextScheduledRun
} = require('../jobs/scraperJob');
const { runScrape } = require('../services/scraperService');
const scraperConfig = require('../config/scraper');
/**
* POST /api/admin/scraper/run
@ -1206,5 +1210,45 @@ router.post('/scraper/run', async (req, res) => {
}
});
/**
* GET /api/admin/scraper/status
* Get current scraper status including running state, last run details,
* next scheduled run, and schedule expression.
*/
router.get('/scraper/status', async (req, res) => {
try {
const db = req.app.locals.db;
// Get last run from history
const lastRunDoc = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.findOne({}, { sort: { startedAt: -1 } });
const lastRun = lastRunDoc ? {
jobId: lastRunDoc.jobId,
timestamp: lastRunDoc.startedAt,
status: lastRunDoc.status,
duration: lastRunDoc.duration,
trigger: lastRunDoc.trigger,
unitsProcessed: lastRunDoc.unitsProcessed,
pricesInserted: lastRunDoc.pricesInserted,
errors: lastRunDoc.errors?.length > 0 ? lastRunDoc.errors : null
} : null;
res.json({
data: {
currentStatus: isScraperRunning() ? 'running' : 'idle',
runningJobId: isScraperRunning() ? getCurrentJobId() : null,
lastRun,
nextScheduledRun: getNextScheduledRun(),
schedule: getScheduleExpression()
}
});
} catch (error) {
console.error('Error fetching scraper status:', error);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
module.exports = router;
module.exports.clearStatsCache = clearStatsCache;