stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

SCRAPE-16: Implement POST /admin/scraper/run endpoint #21

Merged
stephen merged 2 commits from scraper/api-run into dev 2026-02-06 22:07:11 -07:00
Conversation 0 Commits 2 Files Changed 4 +689 -69
4 changed files with 689 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7e796295cb - Show all commits

162
__tests__/scraper/scraperJob.test.js
View File

@ -18,9 +18,10 @@ jest.mock('../../services/scraperLogger', () => ({
}))
}));
// Mock uuid
jest.mock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid-1234')
// Mock crypto.randomUUID
jest.mock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid-1234')
}));
// Mock scraperService
@ -256,8 +257,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -293,8 +295,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -325,8 +328,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -369,8 +373,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -410,8 +415,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -446,8 +452,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -481,8 +488,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
@ -518,8 +526,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const freshCron = require('node-cron');
@ -564,8 +573,9 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'scheduled-job-id')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'scheduled-job-id')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -609,8 +619,9 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-release-test')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'job-release-test')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -649,8 +660,9 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn()
}))
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-error-test')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'job-error-test')
}));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -688,8 +700,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'job-skip-test')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'job-skip-test')
}));
const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
@ -731,8 +744,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
@ -770,8 +784,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -802,8 +817,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const {
@ -855,8 +871,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'running-job-id')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'running-job-id')
}));
const {
@ -925,8 +942,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger)
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'timeout-job-id')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'timeout-job-id')
}));
const {
@ -975,8 +993,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1015,8 +1034,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1047,8 +1067,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const {
@ -1089,8 +1110,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@ -1118,8 +1140,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@ -1166,8 +1189,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob');
@ -1195,8 +1219,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1228,8 +1253,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { gracefulShutdown } = require('../../jobs/scraperJob');
@ -1256,8 +1282,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { isShuttingDown } = require('../../jobs/scraperJob');
@ -1284,8 +1311,9 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn()
}));
jest.doMock('uuid', () => ({
v4: jest.fn(() => 'mock-uuid')
jest.doMock('crypto', () => ({
...jest.requireActual('crypto'),
randomUUID: jest.fn(() => 'mock-uuid')
}));
const { registerSignalHandlers } = require('../../jobs/scraperJob');
@ -1307,7 +1335,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
@ -1327,7 +1355,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
@ -1349,7 +1377,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1369,7 +1397,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1393,7 +1421,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1421,7 +1449,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1451,7 +1479,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1476,7 +1504,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: utcInit, getNextScheduledRun: utcGetNext } = require('../../jobs/scraperJob');
@ -1507,7 +1535,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1539,7 +1567,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
}));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');

4
jobs/scraperJob.js
View File

@ -1,8 +1,8 @@
const cron = require('node-cron');
const crypto = require('crypto');
const config = require('../config/scraper');
const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger');
const { v4: uuidv4 } = require('uuid');
// In-process mutex state
let isRunning = false;
@ -140,7 +140,7 @@ function initializeScheduler(db) {
// Create the scheduled job
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
const jobId = uuidv4();
const jobId = crypto.randomUUID();
const jobLogger = createLogger(jobId);
jobLogger.info('Scheduled scrape triggered');