stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

Compare commits

base: stephen:1c89181d873fdcbffdda4b0f8eff1b487e5f85fe
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field
..
compare: stephen:2b36ef4e238526c225d76c4e133a349e37719050
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field

1 Commits
1c89181d87 ... 2b36ef4e23

Author SHA1 Message Date
Stephen Minakian
2b36ef4e23 feat: add GET /api/admin/scraper/status endpoint
Some checks failed
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 12s
Details
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 40s
Details
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Failing after 1s
Details
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
Details
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Details 
Implement a status endpoint that returns the current state of the
scraper system. The response includes mutex state (idle/running with
job ID), last run history from scraper_runs collection (status,
timing, unit/error counts), next scheduled run timestamp, and
cron schedule expression.

Protected by requireAuth + requireAdmin middleware. Returns 503
on database errors for graceful degradation. Includes 13 tests
covering auth, response structure, edge cases, and error handling.
 2026-02-06 21:30:40 -07:00
4 changed files with 79 additions and 637 deletions
Expand all files Collapse all files

162
__tests__/scraper/scraperJob.test.js
View File

@ -18,10 +18,9 @@ jest.mock('../../services/scraperLogger', () => ({
})) }))
})); }));
// Mock crypto.randomUUID // Mock uuid
jest.mock('crypto', () => ({ jest.mock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid-1234')
randomUUID: jest.fn(() => 'mock-uuid-1234')
})); }));
// Mock scraperService // Mock scraperService
@ -257,9 +256,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -295,9 +293,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -328,9 +325,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -373,9 +369,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -415,9 +410,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -452,9 +446,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -488,9 +481,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { initializeScheduler: freshInit } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit } = require('../../jobs/scraperJob');
@ -526,9 +518,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const freshCron = require('node-cron'); const freshCron = require('node-cron');
@ -573,9 +564,8 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn() info: jest.fn(), warn: jest.fn(), error: jest.fn()
})) }))
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'scheduled-job-id')
randomUUID: jest.fn(() => 'scheduled-job-id')
})); }));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -619,9 +609,8 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn() info: jest.fn(), warn: jest.fn(), error: jest.fn()
})) }))
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'job-release-test')
randomUUID: jest.fn(() => 'job-release-test')
})); }));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -660,9 +649,8 @@ describe('scraperJob - scheduler', () => {
info: jest.fn(), warn: jest.fn(), error: jest.fn() info: jest.fn(), warn: jest.fn(), error: jest.fn()
})) }))
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'job-error-test')
randomUUID: jest.fn(() => 'job-error-test')
})); }));
const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, isScraperRunning: freshIsRunning } = require('../../jobs/scraperJob');
@ -700,9 +688,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({ jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger) createLogger: jest.fn(() => freshLogger)
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'job-skip-test')
randomUUID: jest.fn(() => 'job-skip-test')
})); }));
const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, acquireLock: freshAcquire } = require('../../jobs/scraperJob');
@ -744,9 +731,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, stopScheduler: freshStop } = require('../../jobs/scraperJob');
@ -784,9 +770,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -817,9 +802,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { const {
@ -871,9 +855,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({ jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger) createLogger: jest.fn(() => freshLogger)
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'running-job-id')
randomUUID: jest.fn(() => 'running-job-id')
})); }));
const { const {
@ -942,9 +925,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperLogger', () => ({ jest.doMock('../../services/scraperLogger', () => ({
createLogger: jest.fn(() => freshLogger) createLogger: jest.fn(() => freshLogger)
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'timeout-job-id')
randomUUID: jest.fn(() => 'timeout-job-id')
})); }));
const { const {
@ -993,9 +975,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob'); const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1034,9 +1015,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob'); const { gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1067,9 +1047,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { const {
@ -1110,9 +1089,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob'); const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@ -1140,9 +1118,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob'); const { registerSignalHandlers: freshRegister } = require('../../jobs/scraperJob');
@ -1189,9 +1166,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob'); const { isShuttingDown: freshIsShuttingDown } = require('../../jobs/scraperJob');
@ -1219,9 +1195,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob'); const { isShuttingDown: freshIsShuttingDown, gracefulShutdown: freshShutdown } = require('../../jobs/scraperJob');
@ -1253,9 +1228,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { gracefulShutdown } = require('../../jobs/scraperJob'); const { gracefulShutdown } = require('../../jobs/scraperJob');
@ -1282,9 +1256,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { isShuttingDown } = require('../../jobs/scraperJob'); const { isShuttingDown } = require('../../jobs/scraperJob');
@ -1311,9 +1284,8 @@ describe('scraperJob - scheduler', () => {
jest.doMock('../../services/scraperService', () => ({ jest.doMock('../../services/scraperService', () => ({
runScrape: jest.fn() runScrape: jest.fn()
})); }));
jest.doMock('crypto', () => ({ jest.doMock('uuid', () => ({
...jest.requireActual('crypto'), v4: jest.fn(() => 'mock-uuid')
randomUUID: jest.fn(() => 'mock-uuid')
})); }));
const { registerSignalHandlers } = require('../../jobs/scraperJob'); const { registerSignalHandlers } = require('../../jobs/scraperJob');
@ -1335,7 +1307,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob'); const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
@ -1355,7 +1327,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob'); const { getScheduleExpression: freshGetExpr } = require('../../jobs/scraperJob');
@ -1377,7 +1349,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1397,7 +1369,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1421,7 +1393,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1449,7 +1421,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1479,7 +1451,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1504,7 +1476,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: utcInit, getNextScheduledRun: utcGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: utcInit, getNextScheduledRun: utcGetNext } = require('../../jobs/scraperJob');
@ -1535,7 +1507,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');
@ -1567,7 +1539,7 @@ describe('scraperJob - scheduler', () => {
createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })) createLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() }))
})); }));
jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() })); jest.doMock('../../services/scraperService', () => ({ runScrape: jest.fn() }));
jest.doMock('crypto', () => ({ ...jest.requireActual('crypto'), randomUUID: jest.fn(() => 'mock-uuid') })); jest.doMock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob'); const { initializeScheduler: freshInit, getNextScheduledRun: freshGetNext } = require('../../jobs/scraperJob');

482
__tests__/scraper/scraperRoutes.test.js
View File

@ -1,17 +1,7 @@
/** /**
* Tests for scraper admin route endpoints * Tests for scraper admin route endpoints
* *
* Covers: * Tests GET /api/admin/scraper/status endpoint for:
* - POST /api/admin/scraper/run
* - Authentication and authorization (401, 403)
* - Successful scrape trigger (202 with jobId)
* - Conflict when scraper already running (409)
* - Mutex lock acquisition and release
* - Activity logging for admin trigger
* - dryRun option support
* - htmlContent option support
* - Response structure validation
* - GET /api/admin/scraper/status
* - Authentication and authorization (401/403) * - Authentication and authorization (401/403)
* - Correct status response structure * - Correct status response structure
* - Idle vs running state * - Idle vs running state
@ -40,29 +30,14 @@ jest.mock('../../jobs/scraperJob', () => {
return { return {
isScraperRunning: jest.fn(() => _isRunning), isScraperRunning: jest.fn(() => _isRunning),
acquireLock: jest.fn((jobId) => {
if (_isRunning) return false;
_isRunning = true;
_currentJobId = jobId;
return true;
}),
releaseLock: jest.fn(() => {
_isRunning = false;
_currentJobId = null;
}),
getCurrentJobId: jest.fn(() => _currentJobId), getCurrentJobId: jest.fn(() => _currentJobId),
getScheduleExpression: jest.fn(() => _scheduleExpression), getScheduleExpression: jest.fn(() => _scheduleExpression),
getNextScheduledRun: jest.fn(() => _nextScheduledRun), getNextScheduledRun: jest.fn(() => _nextScheduledRun),
acquireLock: jest.fn(),
releaseLock: jest.fn(),
initializeScheduler: jest.fn(), initializeScheduler: jest.fn(),
stopScheduler: jest.fn(), stopScheduler: jest.fn(),
isShuttingDown: jest.fn(() => false),
gracefulShutdown: jest.fn(),
registerSignalHandlers: jest.fn(),
// Test helpers to control mock state // Test helpers to control mock state
_setState: (running, jobId) => {
_isRunning = running;
_currentJobId = jobId || null;
},
__setRunning: (running, jobId) => { __setRunning: (running, jobId) => {
_isRunning = running; _isRunning = running;
_currentJobId = jobId || null; _currentJobId = jobId || null;
@ -80,20 +55,6 @@ jest.mock('../../jobs/scraperJob', () => {
}; };
}); });
// Mock scraperService to prevent actual scraping
jest.mock('../../services/scraperService', () => ({
runScrape: jest.fn(() => Promise.resolve({ status: 'success', jobId: 'test-job-id' }))
}));
// Mock activityLogger to track calls
jest.mock('../../services/activityLogger', () => {
const originalModule = jest.requireActual('../../services/activityLogger');
return {
...originalModule,
logActivity: jest.fn(() => Promise.resolve())
};
});
// Mock the scraper config for collection names // Mock the scraper config for collection names
jest.mock('../../config/scraper', () => ({ jest.mock('../../config/scraper', () => ({
TARGET_URL: 'https://example.com/test', TARGET_URL: 'https://example.com/test',
@ -113,8 +74,6 @@ jest.mock('../../config/scraper', () => ({
})); }));
const scraperJob = require('../../jobs/scraperJob'); const scraperJob = require('../../jobs/scraperJob');
const { runScrape } = require('../../services/scraperService');
const { logActivity } = require('../../services/activityLogger');
describe('Scraper Routes', () => { describe('Scraper Routes', () => {
let connection; let connection;
@ -124,12 +83,10 @@ describe('Scraper Routes', () => {
beforeAll(async () => { beforeAll(async () => {
const uri = process.env.MONGO_URI; const uri = process.env.MONGO_URI;
connection = await MongoClient.connect(uri); connection = await MongoClient.connect(uri);
db = connection.db('apartments_scraper_routes_test'); db = connection.db('apartments_test');
}); });
afterAll(async () => { afterAll(async () => {
// Allow pending async operations (e.g., .finally() callbacks) to complete
await new Promise(resolve => setTimeout(resolve, 200));
if (connection) { if (connection) {
await connection.close(); await connection.close();
} }
@ -140,438 +97,7 @@ describe('Scraper Routes', () => {
// Also clean scraper_runs collection // Also clean scraper_runs collection
await db.collection('scraper_runs').deleteMany({}); await db.collection('scraper_runs').deleteMany({});
app = await createTestApp(db); app = await createTestApp(db);
// Reset scraper state
scraperJob.__reset(); scraperJob.__reset();
// Clear mock call history (but keep implementations intact)
scraperJob.isScraperRunning.mockClear();
scraperJob.acquireLock.mockClear();
scraperJob.releaseLock.mockClear();
runScrape.mockClear();
logActivity.mockClear();
// Restore default mock implementations
runScrape.mockImplementation(() => Promise.resolve({ status: 'success', jobId: 'test-job-id' }));
});
// ============================================================
// POST /api/admin/scraper/run
// ============================================================
describe('POST /api/admin/scraper/run', () => {
// ============================================================
// Authentication Tests
// ============================================================
describe('authentication and authorization', () => {
it('should return 401 without auth token', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.send({});
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
});
it('should return 403 for non-admin user', async () => {
const regularUser = createTestUser({ role: 'user' });
await insertTestUser(db, regularUser);
const token = generateTestToken(regularUser._id);
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${token}`])
.send({});
expect(res.status).toBe(403);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/admin/i);
});
});
// ============================================================
// Successful Trigger Tests
// ============================================================
describe('successful scrape trigger', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should return 202 with jobId for admin user', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
expect(res.body).toHaveProperty('data');
expect(res.body.data).toHaveProperty('jobId');
expect(res.body.data).toHaveProperty('status', 'started');
expect(res.body.data).toHaveProperty('message');
});
it('should return response with correct structure (jobId, status, message, dryRun)', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
const { data } = res.body;
expect(data.jobId).toBeDefined();
expect(typeof data.jobId).toBe('string');
expect(data.jobId.length).toBeGreaterThan(0);
expect(data.status).toBe('started');
expect(typeof data.message).toBe('string');
expect(data.dryRun).toBe(false);
});
it('should return a UUID-formatted jobId', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// UUID v4 format: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
expect(res.body.data.jobId).toMatch(uuidRegex);
});
});
// ============================================================
// Conflict Tests (409)
// ============================================================
describe('scraper already running', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should return 409 when scraper is already running', async () => {
// Set scraper as running
scraperJob._setState(true, 'existing-job-id');
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(409);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/already in progress/i);
});
it('should check isScraperRunning before starting', async () => {
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(scraperJob.isScraperRunning).toHaveBeenCalled();
});
});
// ============================================================
// Lock Acquisition Tests
// ============================================================
describe('mutex lock behavior', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should acquire lock before starting scrape', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
expect(scraperJob.acquireLock).toHaveBeenCalledWith(expect.any(String));
});
it('should release lock after scrape completes', async () => {
// Make runScrape resolve quickly
runScrape.mockImplementation(() => Promise.resolve({ status: 'success' }));
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// Wait for the async scrape to complete
await new Promise(resolve => setTimeout(resolve, 100));
// The lock should be released via the .finally() callback
expect(scraperJob.releaseLock).toHaveBeenCalled();
});
it('should release lock even when scrape fails', async () => {
// Make runScrape reject
runScrape.mockImplementation(() => Promise.reject(new Error('Scrape failed')));
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// Wait for the async scrape to complete
await new Promise(resolve => setTimeout(resolve, 100));
// Lock should still be released in finally block
expect(scraperJob.releaseLock).toHaveBeenCalled();
});
});
// ============================================================
// Activity Logging Tests
// ============================================================
describe('activity logging', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should log ADMIN_TRIGGER_SCRAPE activity', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
expect(logActivity).toHaveBeenCalledWith(
expect.anything(), // db
expect.objectContaining({
userId: adminUser._id.toString(),
action: 'ADMIN_TRIGGER_SCRAPE',
metadata: expect.objectContaining({
jobId: expect.any(String),
dryRun: false
})
})
);
});
it('should log dryRun flag in activity metadata', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({ dryRun: true });
expect(res.status).toBe(202);
expect(logActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
action: 'ADMIN_TRIGGER_SCRAPE',
metadata: expect.objectContaining({
dryRun: true
})
})
);
});
});
// ============================================================
// dryRun Option Tests
// ============================================================
describe('dryRun option', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should handle dryRun option in request body', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({ dryRun: true });
expect(res.status).toBe(202);
expect(res.body.data.dryRun).toBe(true);
expect(res.body.data.message).toMatch(/dry run/i);
});
it('should pass dryRun option to runScrape', async () => {
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({ dryRun: true });
// Wait for async execution
await new Promise(resolve => setTimeout(resolve, 100));
expect(runScrape).toHaveBeenCalledWith(
expect.anything(), // db
expect.objectContaining({
dryRun: true
})
);
});
it('should default dryRun to false when not provided', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
expect(res.body.data.dryRun).toBe(false);
});
});
// ============================================================
// htmlContent Option Tests
// ============================================================
describe('htmlContent option', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should pass htmlContent option to runScrape', async () => {
const testHtml = '<html><body>Test content</body></html>';
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({ htmlContent: testHtml });
// Wait for async execution
await new Promise(resolve => setTimeout(resolve, 100));
expect(runScrape).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
htmlContent: testHtml
})
);
});
it('should log usingProvidedHtml in activity metadata when htmlContent provided', async () => {
const testHtml = '<html><body>Test</body></html>';
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({ htmlContent: testHtml });
expect(logActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
metadata: expect.objectContaining({
usingProvidedHtml: true
})
})
);
});
});
// ============================================================
// runScrape Integration Tests
// ============================================================
describe('scrape execution', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
it('should call runScrape with trigger "manual"', async () => {
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
// Wait for async execution
await new Promise(resolve => setTimeout(resolve, 100));
expect(runScrape).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
trigger: 'manual'
})
);
});
it('should call runScrape with the generated jobId', async () => {
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
const { jobId } = res.body.data;
// Wait for async execution
await new Promise(resolve => setTimeout(resolve, 100));
expect(runScrape).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
jobId
})
);
});
it('should return 202 immediately without waiting for scrape to finish', async () => {
// Make runScrape take a "long time" (500ms is enough to prove async behavior)
let scrapeResolve;
runScrape.mockImplementation(() => new Promise(resolve => {
scrapeResolve = resolve;
setTimeout(() => resolve({ status: 'success' }), 500);
}));
const startTime = Date.now();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
const elapsed = Date.now() - startTime;
// Should respond in well under 200ms despite scrape taking 500ms
expect(res.status).toBe(202);
expect(elapsed).toBeLessThan(200);
// Resolve the scrape to clean up
if (scrapeResolve) scrapeResolve({ status: 'success' });
await new Promise(resolve => setTimeout(resolve, 100));
});
});
}); });
// ============================================================ // ============================================================

4
jobs/scraperJob.js
View File

@ -1,8 +1,8 @@
const cron = require('node-cron'); const cron = require('node-cron');
const crypto = require('crypto');
const config = require('../config/scraper'); const config = require('../config/scraper');
const { runScrape } = require('../services/scraperService'); const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger'); const { createLogger } = require('../services/scraperLogger');
const { v4: uuidv4 } = require('uuid');
// In-process mutex state // In-process mutex state
let isRunning = false; let isRunning = false;
@ -140,7 +140,7 @@ function initializeScheduler(db) {
// Create the scheduled job // Create the scheduled job
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => { scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
const jobId = crypto.randomUUID(); const jobId = uuidv4();
const jobLogger = createLogger(jobId); const jobLogger = createLogger(jobId);
jobLogger.info('Scheduled scrape triggered'); jobLogger.info('Scheduled scrape triggered');

56
routes/admin.js
View File

@ -1146,70 +1146,14 @@ router.patch('/settings', async (req, res) => {
// Scraper Endpoints // Scraper Endpoints
// ============================================================ // ============================================================
const crypto = require('crypto');
const { const {
isScraperRunning, isScraperRunning,
acquireLock,
releaseLock,
getCurrentJobId, getCurrentJobId,
getScheduleExpression, getScheduleExpression,
getNextScheduledRun getNextScheduledRun
} = require('../jobs/scraperJob'); } = require('../jobs/scraperJob');
const { runScrape } = require('../services/scraperService');
const scraperConfig = require('../config/scraper'); const scraperConfig = require('../config/scraper');
/**
* POST /api/admin/scraper/run
* Trigger a manual scrape
*
* Request body (optional):
* - dryRun: boolean - Skip database writes for safe testing
* - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
*/
router.post('/scraper/run', async (req, res) => {
try {
const db = req.app.locals.db;
const { dryRun = false, htmlContent = null } = req.body || {};
// Check if scraper is already running
if (isScraperRunning()) {
return res.status(409).json({ error: 'Scrape already in progress' });
}
// Generate job ID and acquire lock
const jobId = crypto.randomUUID();
acquireLock(jobId);
// Log admin action
await logActivity(db, {
userId: req.user._id.toString(),
action: 'ADMIN_TRIGGER_SCRAPE',
metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
});
// Start scrape asynchronously (do not await - return 202 immediately)
runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
.catch((error) => {
console.error('Async scrape failed:', error.message);
})
.finally(() => releaseLock());
// Return immediately with job ID
res.status(202).json({
data: {
jobId,
status: 'started',
dryRun,
message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
}
});
} catch (error) {
console.error('Error triggering scrape:', error);
res.status(500).json({ error: 'Failed to start scrape job' });
}
});
/** /**
* GET /api/admin/scraper/status * GET /api/admin/scraper/status
* Get current scraper status including running state, last run details, * Get current scraper status including running state, last run details,