stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

Compare commits

base: stephen:dev
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field
..
compare: stephen:1c89181d873fdcbffdda4b0f8eff1b487e5f85fe
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field

1 Commits
dev ... 1c89181d87

Author SHA1 Message Date
Stephen Minakian
1c89181d87 feat: add GET /api/admin/scraper/status endpoint
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
Details
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
Details
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 41s
Details
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
Details
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Details 
Implement a status endpoint that returns the current state of the
scraper system. The response includes mutex state (idle/running with
job ID), last run history from scraper_runs collection (status,
timing, unit/error counts), next scheduled run timestamp, and
cron schedule expression.

Protected by requireAuth + requireAdmin middleware. Returns 503
on database errors for graceful degradation. Includes 13 tests
covering auth, response structure, edge cases, and error handling.
 2026-02-06 22:12:47 -07:00
16 changed files with 260 additions and 2219 deletions
Expand all files Collapse all files

25
.github/workflows/deploy.yml vendored
View File

@ -44,10 +44,8 @@ jobs:
set +e set +e
OUTPUT=$(npm run lint 2>&1) OUTPUT=$(npm run lint 2>&1)
EXIT_CODE=$? EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "lint_output<<EOF" >> $GITHUB_OUTPUT echo "lint_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE exit $EXIT_CODE
@ -58,10 +56,8 @@ jobs:
set +e set +e
OUTPUT=$(npm test -- --runInBand 2>&1) OUTPUT=$(npm test -- --runInBand 2>&1)
EXIT_CODE=$? EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "test_output<<EOF" >> $GITHUB_OUTPUT echo "test_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE exit $EXIT_CODE
env: env:
@ -102,9 +98,9 @@ jobs:
OVERALL_STATUS="failure" OVERALL_STATUS="failure"
fi fi
# Outputs are already truncated at the source (ci job) # Truncate outputs if too long (max 10000 chars each)
LINT_OUTPUT="$RAW_LINT_OUTPUT" LINT_OUTPUT="${RAW_LINT_OUTPUT:0:10000}"
TEST_OUTPUT="$RAW_TEST_OUTPUT" TEST_OUTPUT="${RAW_TEST_OUTPUT:0:10000}"
# Build JSON payload # Build JSON payload
PAYLOAD=$(jq -n \ PAYLOAD=$(jq -n \
@ -264,13 +260,12 @@ jobs:
# ============================================================ # ============================================================
# SBOM Generation with Syft # SBOM Generation with Syft
# ============================================================ # ============================================================
- name: Install Syft
run: |
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
- name: Generate SBOM with Syft - name: Generate SBOM with Syft
run: | uses: anchore/sbom-action@v0
syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json with:
image: ${{ steps.set-tag.outputs.full_image }}
format: spdx-json
output-file: sbom.spdx.json
# ============================================================ # ============================================================
# Image Signing with Cosign # Image Signing with Cosign

64
__tests__/scraper/fixtures/sample-listing-call.html
View File

@ -1,64 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - Call for Pricing</title></head>
<body>
<section class="spaces__tab-unit">
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9001"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="750"
data-spaces-sort-bed="1"
data-spaces-unit="P-101"
data-spaces-unit-id="9001"
data-spaces-unit-floor="1"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="500"
data-spaces-sort-plan-name="Penthouse A"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=9001"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
aria-label="Unit P-101">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-101</span>
</div>
</article>
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9002"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="900"
data-spaces-sort-bed="2"
data-spaces-unit="P-102"
data-spaces-unit-id="9002"
data-spaces-unit-floor="2"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="501"
data-spaces-sort-plan-name="Penthouse B"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=9002"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
aria-label="Unit P-102">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-102</span>
</div>
</article>
</section>
</body>
</html>

9
__tests__/scraper/fixtures/sample-listing-empty.html
View File

@ -1,9 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - No Units</title></head>
<body>
<section class="spaces__tab-unit">
<!-- Empty - no units available -->
</section>
</body>
</html>

424
__tests__/scraper/fixtures/sample-listing.html
View File

@ -1,288 +1,144 @@
<!DOCTYPE html> <!DOCTYPE html>
<html> <html lang="en">
<head><title>Country Club Towers - Apartments</title></head> <head>
<title>Country Club Towers - Available Units</title>
</head>
<body> <body>
<section class="spaces__tab-unit"> <div class="spaces-container">
<article <section class="spaces__tab-unit">
class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens" <!-- Unit 1: Fully populated with all attributes and an image -->
data-spaces-id="154842" <article
data-spaces-soonest="2025-10-11" data-spaces-id="1001"
data-spaces-sort-date="1760140800" data-spaces-unit="CCT-101"
data-spaces-sort-price="5230" data-spaces-unit-id="5001"
data-spaces-sort-area="1210" data-spaces-unit-floor="1"
data-spaces-sort-bed="2" data-spaces-sort-area="750"
data-spaces-unit="W2707" data-spaces-sort-bed="1"
data-spaces-unit-id="154842" data-spaces-sort-bath="1"
data-spaces-unit-floor="2755" data-spaces-sort-price="1450"
data-spaces-obj="unit" data-spaces-available="true"
data-spaces-unavailable="false" data-spaces-unavailable="false"
data-spaces-available="true" data-spaces-soonest="2026-03-01"
data-spaces-community="Country Club Towers" data-spaces-sort-date="20260301"
data-spaces-asset="420" data-spaces-plan-id="201"
data-spaces-specials-content="" data-spaces-sort-plan-name="Alpine"
data-spaces-plan-id="10270" data-spaces-obj="unit"
data-spaces-sort-plan-name="Pyramid Peak - Terrace" data-spaces-community="Country Club Towers"
data-spaces-sort-bath="2" data-spaces-asset="100"
data-spaces-href="?spaces_tab=unit-detail&detail=154842" data-spaces-href="/units/CCT-101"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842" data-spaces-inventory-href="/inventory/CCT-101"
aria-label="Unit W2707"> data-spaces-specials-content="First month free!"
<div class="spaces-unit__inner"> >
<span class="spaces-unit__price">5230</span> <div class="unit-card">
<span class="spaces-unit__unit">W2707</span> <img src="https://example.com/images/unit-101.jpg" alt="Unit 101" />
</div> <h3>Unit CCT-101</h3>
</article> <p>1 Bed / 1 Bath - $1,450/mo</p>
<article </div>
class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens" </article>
data-spaces-id="154451"
data-spaces-soonest="2026-03-08" <!-- Unit 2: Different floor plan, no specials, with data-src image -->
data-spaces-sort-date="1772928000" <article
data-spaces-sort-price="2767" data-spaces-id="1002"
data-spaces-sort-area="806" data-spaces-unit="CCT-205"
data-spaces-sort-bed="1" data-spaces-unit-id="5002"
data-spaces-unit="E0901" data-spaces-unit-floor="2"
data-spaces-unit-id="154451" data-spaces-sort-area="950"
data-spaces-unit-floor="2738" data-spaces-sort-bed="2"
data-spaces-obj="unit" data-spaces-sort-bath="1.5"
data-spaces-unavailable="false" data-spaces-sort-price="1850"
data-spaces-available="true" data-spaces-available="true"
data-spaces-community="Country Club Towers" data-spaces-unavailable="false"
data-spaces-asset="420" data-spaces-soonest="2026-02-15"
data-spaces-specials-content="" data-spaces-sort-date="20260215"
data-spaces-plan-id="8016" data-spaces-plan-id="202"
data-spaces-sort-plan-name="Sunshine Peak" data-spaces-sort-plan-name="Birchwood"
data-spaces-sort-bath="1" data-spaces-obj="unit"
data-spaces-href="?spaces_tab=unit-detail&detail=154451" data-spaces-community="Country Club Towers"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451" data-spaces-asset="100"
aria-label="Unit E0901"> data-spaces-href="/units/CCT-205"
<div class="spaces-unit__inner"> data-spaces-inventory-href="/inventory/CCT-205"
<span class="spaces-unit__price">2767</span> data-spaces-specials-content=""
<span class="spaces-unit__unit">E0901</span> >
</div> <div class="unit-card">
</article> <img data-src="https://example.com/images/unit-205.jpg" alt="Unit 205" />
<article <h3>Unit CCT-205</h3>
class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens" </div>
data-spaces-id="154814" </article>
data-spaces-soonest="2025-12-18"
data-spaces-sort-date="1766016000" <!-- Unit 3: Studio with some missing attributes -->
data-spaces-sort-price="2255" <article
data-spaces-sort-area="623" data-spaces-id="1003"
data-spaces-sort-bed="0" data-spaces-unit="CCT-310"
data-spaces-unit="W2603" data-spaces-unit-id="5003"
data-spaces-unit-id="154814" data-spaces-unit-floor="3"
data-spaces-unit-floor="2754" data-spaces-sort-area="500"
data-spaces-obj="unit" data-spaces-sort-bed="0"
data-spaces-unavailable="false" data-spaces-sort-bath="1"
data-spaces-available="true" data-spaces-sort-price="1100"
data-spaces-community="Country Club Towers" data-spaces-available="true"
data-spaces-asset="420" data-spaces-unavailable="false"
data-spaces-specials-content="" data-spaces-soonest="Available Now"
data-spaces-plan-id="8018" data-spaces-sort-date="20260201"
data-spaces-sort-plan-name="Little Bear Peak" data-spaces-plan-id="203"
data-spaces-sort-bath="1" data-spaces-sort-plan-name="Cedar"
data-spaces-href="?spaces_tab=unit-detail&detail=154814" data-spaces-obj="unit"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814" data-spaces-community="Country Club Towers"
aria-label="Unit W2603"> data-spaces-asset="100"
<div class="spaces-unit__inner"> data-spaces-href="/units/CCT-310"
<span class="spaces-unit__price">2255</span> data-spaces-inventory-href="/inventory/CCT-310"
<span class="spaces-unit__unit">W2603</span> >
</div> <div class="unit-card">
</article> <h3>Unit CCT-310</h3>
<article <p>Studio - $1,100/mo</p>
class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens" </div>
data-spaces-id="154428" </article>
data-spaces-soonest="2026-02-22"
data-spaces-sort-date="1771718400" <!-- Unit 4: Unavailable unit -->
data-spaces-sort-price="4250" <article
data-spaces-sort-area="1152" data-spaces-id="1004"
data-spaces-sort-bed="2" data-spaces-unit="CCT-415"
data-spaces-unit="W0802" data-spaces-unit-id="5004"
data-spaces-unit-id="154428" data-spaces-unit-floor="4"
data-spaces-unit-floor="2737" data-spaces-sort-area="1200"
data-spaces-obj="unit" data-spaces-sort-bed="3"
data-spaces-unavailable="false" data-spaces-sort-bath="2"
data-spaces-available="true" data-spaces-sort-price="2500"
data-spaces-community="Country Club Towers" data-spaces-available="false"
data-spaces-asset="420" data-spaces-unavailable="true"
data-spaces-specials-content="" data-spaces-soonest=""
data-spaces-plan-id="8017" data-spaces-sort-date=""
data-spaces-sort-plan-name="Mt. Princeton" data-spaces-plan-id="204"
data-spaces-sort-bath="2" data-spaces-sort-plan-name="Dogwood"
data-spaces-href="?spaces_tab=unit-detail&detail=154428" data-spaces-obj="unit"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428" data-spaces-community="Country Club Towers"
aria-label="Unit W0802"> data-spaces-asset="100"
<div class="spaces-unit__inner"> data-spaces-href="/units/CCT-415"
<span class="spaces-unit__price">4250</span> data-spaces-inventory-href="/inventory/CCT-415"
<span class="spaces-unit__unit">W0802</span> data-spaces-specials-content=""
</div> >
</article> <div class="unit-card">
<article <img src="https://example.com/images/unit-415.jpg" alt="Unit 415" />
class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens" <h3>Unit CCT-415</h3>
data-spaces-id="154778" </div>
data-spaces-soonest="2025-12-31" </article>
data-spaces-sort-date="1767139200"
data-spaces-sort-price="2495" <!-- Unit 5: Minimal attributes - edge case -->
data-spaces-sort-area="764" <article
data-spaces-sort-bed="1" data-spaces-id="1005"
data-spaces-unit="W2405" data-spaces-unit="CCT-520"
data-spaces-unit-id="154778" data-spaces-unit-id="5005"
data-spaces-unit-floor="2752" data-spaces-sort-price="1300"
data-spaces-obj="unit" data-spaces-available="true"
data-spaces-unavailable="false" data-spaces-unavailable="false"
data-spaces-available="true" data-spaces-obj="unit"
data-spaces-community="Country Club Towers" data-spaces-community="Country Club Towers"
data-spaces-asset="420" data-spaces-asset="100"
data-spaces-specials-content="" >
data-spaces-plan-id="8020" <div class="unit-card">
data-spaces-sort-plan-name="Grays Peak" <h3>Unit CCT-520</h3>
data-spaces-sort-bath="1" </div>
data-spaces-href="?spaces_tab=unit-detail&detail=154778" </article>
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778" </section>
aria-label="Unit W2405"> </div>
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W2405</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154442"
data-spaces-soonest="2025-02-01"
data-spaces-sort-date="1738368000"
data-spaces-sort-price="2495"
data-spaces-sort-area="802"
data-spaces-sort-bed="1"
data-spaces-unit="W0809"
data-spaces-unit-id="154442"
data-spaces-unit-floor="2737"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8022"
data-spaces-sort-plan-name="Pikes Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154442"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
aria-label="Unit W0809">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W0809</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154764"
data-spaces-soonest="2026-02-17"
data-spaces-sort-date="1771286400"
data-spaces-sort-price="2435"
data-spaces-sort-area="715"
data-spaces-sort-bed="1"
data-spaces-unit="W2308"
data-spaces-unit-id="154764"
data-spaces-unit-floor="2751"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8023"
data-spaces-sort-plan-name="Longs Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154764"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
aria-label="Unit W2308">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2435</span>
<span class="spaces-unit__unit">W2308</span>
</div>
</article>
<article
class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154612"
data-spaces-soonest="2026-03-24"
data-spaces-sort-date="1774310400"
data-spaces-sort-price="2683"
data-spaces-sort-area="797"
data-spaces-sort-bed="1"
data-spaces-unit="W1610"
data-spaces-unit-id="154612"
data-spaces-unit-floor="2744"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8024"
data-spaces-sort-plan-name="Torreys Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154612"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
aria-label="Unit W1610">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2683</span>
<span class="spaces-unit__unit">W1610</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154495"
data-spaces-soonest="2025-11-07"
data-spaces-sort-date="1762473600"
data-spaces-sort-price="4470"
data-spaces-sort-area="1230"
data-spaces-sort-bed="2"
data-spaces-unit="E1011"
data-spaces-unit-id="154495"
data-spaces-unit-floor="2739"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8025"
data-spaces-sort-plan-name="Maroon Peak"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154495"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
aria-label="Unit E1011">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">4470</span>
<span class="spaces-unit__unit">E1011</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="154919"
data-spaces-soonest="2025-10-27"
data-spaces-sort-date="1761523200"
data-spaces-sort-price="8581"
data-spaces-sort-area="1532"
data-spaces-sort-bed="2"
data-spaces-unit="E3205"
data-spaces-unit-id="154919"
data-spaces-unit-floor="2760"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8037"
data-spaces-sort-plan-name="Mt. Antero"
data-spaces-sort-bath="2.5"
data-spaces-href="?spaces_tab=unit-detail&detail=154919"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
aria-label="Unit E3205">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">8581</span>
<span class="spaces-unit__unit">E3205</span>
</div>
</article>
</section>
</body> </body>
</html> </html>

2
__tests__/scraper/insertPrices.test.js
View File

@ -22,7 +22,7 @@ let mongoServer;
let client; let client;
let db; let db;
const PRICES_COLLECTION = 'unit_prices_scraper'; const PRICES_COLLECTION = 'unit_prices_migration_test';
// Mock logger for capturing log calls // Mock logger for capturing log calls
function createMockLogger() { function createMockLogger() {

2
__tests__/scraper/markStaleUnits.test.js
View File

@ -21,7 +21,7 @@ let mongoServer;
let client; let client;
let db; let db;
const UNITS_COLLECTION = 'units_scraper'; const UNITS_COLLECTION = 'units_migration_test';
// Mock logger for capturing log calls // Mock logger for capturing log calls
function createMockLogger() { function createMockLogger() {

289
__tests__/scraper/parseUnits.test.js
View File

@ -87,55 +87,55 @@ describe('parseUnits', () => {
describe('attribute extraction', () => { describe('attribute extraction', () => {
it('should extract all expected attributes from a fully populated article', () => { it('should extract all expected attributes from a fully populated article', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'W2707'); const unit1 = units.find(u => u.unit_code === 'CCT-101');
expect(unit1).toBeDefined(); expect(unit1).toBeDefined();
// Core identifiers // Core identifiers
expect(unit1.id).toBe('154842'); expect(unit1.id).toBe('1001');
expect(unit1.unit_code).toBe('W2707'); expect(unit1.unit_code).toBe('CCT-101');
expect(unit1.unit_id).toBe('154842'); expect(unit1.unit_id).toBe('5001');
// Physical attributes // Physical attributes
expect(unit1.floor).toBe('2755'); expect(unit1.floor).toBe('1');
expect(unit1.area).toBe('1210'); expect(unit1.area).toBe('750');
expect(unit1.bed_count).toBe('2'); expect(unit1.bed_count).toBe('1');
expect(unit1.bath_count).toBe('2'); expect(unit1.bath_count).toBe('1');
// Pricing // Pricing
expect(unit1.price).toBe('5230'); expect(unit1.price).toBe('1450');
// Availability // Availability
expect(unit1.available).toBe('true'); expect(unit1.available).toBe('true');
expect(unit1.unavailable).toBe('false'); expect(unit1.unavailable).toBe('false');
expect(unit1.soonest).toBe('2025-10-11'); expect(unit1.soonest).toBe('2026-03-01');
expect(unit1.date_available).toBe('1760140800'); expect(unit1.date_available).toBe('20260301');
// Plan information // Plan information
expect(unit1.plan_id).toBe('10270'); expect(unit1.plan_id).toBe('201');
expect(unit1.plan_name).toBe('Pyramid Peak - Terrace'); expect(unit1.plan_name).toBe('Alpine');
// Property information // Property information
expect(unit1.obj_type).toBe('unit'); expect(unit1.obj_type).toBe('unit');
expect(unit1.community).toBe('Country Club Towers'); expect(unit1.community).toBe('Country Club Towers');
expect(unit1.asset).toBe('420'); expect(unit1.asset).toBe('100');
// URLs // URLs
expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842'); expect(unit1.href).toBe('/units/CCT-101');
expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842'); expect(unit1.inventory_href).toBe('/inventory/CCT-101');
// Specials // Specials
expect(unit1.specials_content).toBe(''); expect(unit1.specials_content).toBe('First month free!');
}); });
it('should extract correct count of units from the fixture', () => { it('should extract correct count of units from the fixture', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
expect(units).toHaveLength(10); expect(units).toHaveLength(5);
}); });
it('should return raw string values without type conversion', () => { it('should return raw string values without type conversion', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'W2707'); const unit = units.find(u => u.unit_code === 'CCT-101');
// All values should be strings (raw extraction, no conversion) // All values should be strings (raw extraction, no conversion)
expect(typeof unit.id).toBe('string'); expect(typeof unit.id).toBe('string');
@ -152,48 +152,24 @@ describe('parseUnits', () => {
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('image URL extraction', () => { describe('image URL extraction', () => {
it('should extract image_url from img src attribute', () => { it('should extract image_url from img src attribute', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> const unit1 = units.find(u => u.unit_code === 'CCT-101');
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
<img src="https://example.com/images/unit-1.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg'); expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
}); });
it('should fall back to data-src when src is not present', () => { it('should fall back to data-src when src is not present', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> const unit2 = units.find(u => u.unit_code === 'CCT-205');
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
<img data-src="https://example.com/images/unit-2.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg'); expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
}); });
it('should not set image_url when no img element exists', () => { it('should not set image_url when no img element exists', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> const unit3 = units.find(u => u.unit_code === 'CCT-310');
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
<div>No image here</div>
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBeUndefined(); expect(unit3.image_url).toBeUndefined();
}); });
}); });
@ -202,67 +178,37 @@ describe('parseUnits', () => {
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('missing attributes', () => { describe('missing attributes', () => {
it('should return undefined for attributes not present on the article', () => { it('should return undefined for attributes not present on the article', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> // Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
<section class="spaces__tab-unit"> // soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
<article const unit5 = units.find(u => u.unit_code === 'CCT-520');
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit).toBeDefined(); expect(unit5).toBeDefined();
expect(unit.floor).toBeUndefined(); expect(unit5.floor).toBeUndefined();
expect(unit.area).toBeUndefined(); expect(unit5.area).toBeUndefined();
expect(unit.bed_count).toBeUndefined(); expect(unit5.bed_count).toBeUndefined();
expect(unit.bath_count).toBeUndefined(); expect(unit5.bath_count).toBeUndefined();
expect(unit.soonest).toBeUndefined(); expect(unit5.soonest).toBeUndefined();
expect(unit.date_available).toBeUndefined(); expect(unit5.date_available).toBeUndefined();
expect(unit.plan_id).toBeUndefined(); expect(unit5.plan_id).toBeUndefined();
expect(unit.plan_name).toBeUndefined(); expect(unit5.plan_name).toBeUndefined();
expect(unit.href).toBeUndefined(); expect(unit5.href).toBeUndefined();
expect(unit.inventory_href).toBeUndefined(); expect(unit5.inventory_href).toBeUndefined();
expect(unit.specials_content).toBeUndefined(); expect(unit5.specials_content).toBeUndefined();
}); });
it('should still extract the attributes that are present on a sparse article', () => { it('should still extract the attributes that are present on a sparse article', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> const unit5 = units.find(u => u.unit_code === 'CCT-520');
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit.id).toBe('1005'); expect(unit5.id).toBe('1005');
expect(unit.unit_code).toBe('SPARSE-1'); expect(unit5.unit_code).toBe('CCT-520');
expect(unit.unit_id).toBe('5005'); expect(unit5.unit_id).toBe('5005');
expect(unit.price).toBe('1300'); expect(unit5.price).toBe('1300');
expect(unit.available).toBe('true'); expect(unit5.available).toBe('true');
expect(unit.obj_type).toBe('unit'); expect(unit5.obj_type).toBe('unit');
expect(unit.community).toBe('Country Club Towers'); expect(unit5.community).toBe('Country Club Towers');
expect(unit.asset).toBe('100'); expect(unit5.asset).toBe('100');
}); });
}); });
@ -367,7 +313,7 @@ describe('parseUnits', () => {
expect(mockLogger.info).toHaveBeenCalledWith( expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed', 'Units parsed',
expect.objectContaining({ count: 10 }) expect.objectContaining({ count: 5 })
); );
}); });
@ -384,134 +330,31 @@ describe('parseUnits', () => {
}); });
// --------------------------------------------------------------- // ---------------------------------------------------------------
// 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html) // 7. Multiple units with varied data
// ---------------------------------------------------------------
describe('fixture: sample-listing-empty.html', () => {
let emptyHtml;
beforeAll(() => {
emptyHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-empty.html'),
'utf-8'
);
});
it('should return empty array when fixture has container but no articles', () => {
const units = parseUnits(emptyHtml, mockLogger);
expect(units).toEqual([]);
// Container exists so no error about missing container
expect(mockLogger.error).not.toHaveBeenCalled();
});
it('should log zero parsed units for empty fixture', () => {
parseUnits(emptyHtml, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 0 })
);
});
});
describe('fixture: sample-listing-call.html', () => {
let callHtml;
beforeAll(() => {
callHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-call.html'),
'utf-8'
);
});
it('should extract both units from the call-for-pricing fixture', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units).toHaveLength(2);
expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
});
it('should preserve "Call for pricing" as raw string in the price field', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].price).toBe('Call for pricing');
expect(units[1].price).toBe('Call for pricing');
});
it('should extract all attributes correctly from call-for-pricing units', () => {
const units = parseUnits(callHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'P-101');
expect(unit.id).toBe('9001');
expect(unit.unit_id).toBe('9001');
expect(unit.floor).toBe('1');
expect(unit.area).toBe('750');
expect(unit.bed_count).toBe('1');
expect(unit.bath_count).toBe('1');
expect(unit.available).toBe('true');
expect(unit.unavailable).toBe('false');
expect(unit.soonest).toBe('Now');
expect(unit.date_available).toBe('1706745600');
expect(unit.plan_id).toBe('500');
expect(unit.plan_name).toBe('Penthouse A');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('420');
expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.specials_content).toBe('');
});
it('should not have image_url when fixture units lack img elements', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].image_url).toBeUndefined();
expect(units[1].image_url).toBeUndefined();
});
});
// ---------------------------------------------------------------
// 8. Multiple units with varied data
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('varied unit data', () => { describe('varied unit data', () => {
it('should handle unavailable units correctly', () => { it('should handle unavailable units correctly', () => {
const html = ` const units = parseUnits(sampleHtml, mockLogger);
<html><body> const unit4 = units.find(u => u.unit_code === 'CCT-415');
<section class="spaces__tab-unit">
<article
data-spaces-id="4001"
data-spaces-unit="UNAVAIL-1"
data-spaces-sort-price="2000"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date="">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit.available).toBe('false'); expect(unit4.available).toBe('false');
expect(unit.unavailable).toBe('true'); expect(unit4.unavailable).toBe('true');
expect(unit.soonest).toBe(''); expect(unit4.soonest).toBe('');
expect(unit.date_available).toBe(''); expect(unit4.date_available).toBe('');
}); });
it('should handle half bath counts', () => { it('should handle half bath counts', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
// E3205 is the penthouse with 2.5 baths const unit2 = units.find(u => u.unit_code === 'CCT-205');
const unit = units.find(u => u.unit_code === 'E3205');
expect(unit.bath_count).toBe('2.5'); expect(unit2.bath_count).toBe('1.5');
}); });
it('should handle studio units (bed_count = 0)', () => { it('should handle studio units (bed_count = 0)', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
// W2603 is the studio with bed_count 0 const unit3 = units.find(u => u.unit_code === 'CCT-310');
const unit = units.find(u => u.unit_code === 'W2603');
expect(unit.bed_count).toBe('0'); expect(unit3.bed_count).toBe('0');
}); });
}); });
}); });

16
__tests__/scraper/scraperConfig.test.js
View File

@ -50,15 +50,15 @@ describe('config/scraper', () => {
}); });
describe('SCRAPER_TIMEZONE', () => { describe('SCRAPER_TIMEZONE', () => {
test('should default to "America/Denver" when env var not set', () => { test('should default to "UTC" when env var not set', () => {
const config = require('../../config/scraper'); const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver'); expect(config.SCRAPER_TIMEZONE).toBe('UTC');
}); });
test('should use env var override when set', () => { test('should use env var override when set', () => {
process.env.SCRAPER_TIMEZONE = 'America/New_York'; process.env.SCRAPER_TIMEZONE = 'America/Denver';
const config = require('../../config/scraper'); const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/New_York'); expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
}); });
}); });
@ -142,14 +142,14 @@ describe('config/scraper', () => {
describe('COLLECTIONS', () => { describe('COLLECTIONS', () => {
describe('default values', () => { describe('default values', () => {
test('UNITS should default to "units_scraper"', () => { test('UNITS should default to "units_migration_test"', () => {
const config = require('../../config/scraper'); const config = require('../../config/scraper');
expect(config.COLLECTIONS.UNITS).toBe('units_scraper'); expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
}); });
test('PRICES should default to "unit_prices_scraper"', () => { test('PRICES should default to "unit_prices_migration_test"', () => {
const config = require('../../config/scraper'); const config = require('../../config/scraper');
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper'); expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
}); });
test('DAILY_SUMMARIES should default to "daily_summaries"', () => { test('DAILY_SUMMARIES should default to "daily_summaries"', () => {

898
__tests__/scraper/scraperRoutes.test.js
View File

@ -18,22 +18,9 @@
* - lastRun details from database * - lastRun details from database
* - nextScheduledRun and schedule fields * - nextScheduledRun and schedule fields
* - 503 on database errors * - 503 on database errors
* - GET /api/admin/scraper/history
* - Authentication and authorization (401/403)
* - Default pagination (limit 30, offset 0)
* - Custom limit and offset query params
* - Limit validation (< 1, NaN returns 400)
* - Offset validation (< 0, NaN returns 400)
* - Limit capped at 100
* - Results sorted by startedAt descending
* - Returns array of run records in data field
* - Returns empty array when no runs exist
* - Returns empty array when offset beyond total
* - 503 on database errors
*/ */
const request = require('supertest'); const request = require('supertest');
const jwt = require('jsonwebtoken');
const { MongoClient, ObjectId } = require('mongodb'); const { MongoClient, ObjectId } = require('mongodb');
const { const {
createTestApp, createTestApp,
@ -118,8 +105,8 @@ jest.mock('../../config/scraper', () => ({
RETRY_CONFIG: { maxRetries: 3, baseDelay: 1000, timeout: 30000 }, RETRY_CONFIG: { maxRetries: 3, baseDelay: 1000, timeout: 30000 },
SHUTDOWN_TIMEOUT: 30000, SHUTDOWN_TIMEOUT: 30000,
COLLECTIONS: { COLLECTIONS: {
UNITS: 'units_scraper', UNITS: 'units_migration_test',
PRICES: 'unit_prices_scraper', PRICES: 'unit_prices_migration_test',
DAILY_SUMMARIES: 'daily_summaries', DAILY_SUMMARIES: 'daily_summaries',
SCRAPER_RUNS: 'scraper_runs' SCRAPER_RUNS: 'scraper_runs'
} }
@ -856,885 +843,4 @@ describe('Scraper Routes', () => {
}); });
}); });
}); });
// ============================================================
// GET /api/admin/scraper/history
// ============================================================
describe('GET /api/admin/scraper/history', () => {
describe('Authentication and Authorization', () => {
it('should return 401 without authentication', async () => {
const res = await request(app)
.get('/api/admin/scraper/history')
.expect(401);
expect(res.body).toHaveProperty('error');
});
it('should return 403 for non-admin user', async () => {
const regularUser = createTestUser({ role: 'user' });
await insertTestUser(db, regularUser);
const token = generateTestToken(regularUser._id);
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(403);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Admin access required');
});
});
describe('Successful history response (admin)', () => {
let admin;
let token;
beforeEach(async () => {
admin = createTestAdmin();
await insertTestUser(db, admin);
token = generateTestToken(admin._id);
});
it('should return 200 with data array for admin', async () => {
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body).toHaveProperty('data');
expect(Array.isArray(res.body.data)).toBe(true);
});
it('should return empty array when no runs exist', async () => {
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toEqual([]);
});
it('should return run records with correct fields', async () => {
await db.collection('scraper_runs').insertOne({
jobId: 'history-job-001',
trigger: 'scheduled',
status: 'success',
startedAt: '2026-02-05T06:00:00.000Z',
completedAt: '2026-02-05T06:00:12.345Z',
duration: 12345,
unitsProcessed: 50,
pricesInserted: 48,
newUnitsCount: 2,
rentedUnitsCount: 1,
staleUnitsCount: 0,
errors: [],
recordedAt: new Date('2026-02-05T06:00:12.345Z')
});
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(1);
const record = res.body.data[0];
expect(record.jobId).toBe('history-job-001');
expect(record.trigger).toBe('scheduled');
expect(record.status).toBe('success');
expect(record.startedAt).toBe('2026-02-05T06:00:00.000Z');
expect(record.completedAt).toBe('2026-02-05T06:00:12.345Z');
expect(record.duration).toBe(12345);
expect(record.unitsProcessed).toBe(50);
expect(record.pricesInserted).toBe(48);
expect(record.newUnitsCount).toBe(2);
expect(record.rentedUnitsCount).toBe(1);
expect(record.staleUnitsCount).toBe(0);
expect(record.errors).toEqual([]);
});
it('should sort results by startedAt descending (newest first)', async () => {
// Insert runs in non-chronological order
await db.collection('scraper_runs').insertMany([
{
jobId: 'oldest-job',
trigger: 'scheduled',
status: 'success',
startedAt: '2026-02-03T06:00:00.000Z',
completedAt: '2026-02-03T06:00:10.000Z',
duration: 10000,
unitsProcessed: 40,
pricesInserted: 40,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: new Date('2026-02-03T06:00:10.000Z')
},
{
jobId: 'newest-job',
trigger: 'manual',
status: 'success',
startedAt: '2026-02-05T14:00:00.000Z',
completedAt: '2026-02-05T14:00:08.000Z',
duration: 8000,
unitsProcessed: 52,
pricesInserted: 50,
newUnitsCount: 1,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: new Date('2026-02-05T14:00:08.000Z')
},
{
jobId: 'middle-job',
trigger: 'scheduled',
status: 'failed',
startedAt: '2026-02-04T06:00:00.000Z',
completedAt: '2026-02-04T06:00:32.000Z',
duration: 32000,
unitsProcessed: 0,
pricesInserted: 0,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: ['HTTP request failed'],
recordedAt: new Date('2026-02-04T06:00:32.000Z')
}
]);
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(3);
expect(res.body.data[0].jobId).toBe('newest-job');
expect(res.body.data[1].jobId).toBe('middle-job');
expect(res.body.data[2].jobId).toBe('oldest-job');
});
it('should default limit to 30 records', async () => {
// Insert 35 records
const runs = [];
for (let i = 0; i < 35; i++) {
const date = new Date('2026-01-01T06:00:00.000Z');
date.setDate(date.getDate() + i);
runs.push({
jobId: `job-${String(i).padStart(3, '0')}`,
trigger: 'scheduled',
status: 'success',
startedAt: date.toISOString(),
completedAt: date.toISOString(),
duration: 10000,
unitsProcessed: 50,
pricesInserted: 50,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: date
});
}
await db.collection('scraper_runs').insertMany(runs);
const res = await request(app)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(30);
});
it('should respect custom limit query parameter', async () => {
// Insert 10 records
const runs = [];
for (let i = 0; i < 10; i++) {
const date = new Date('2026-01-01T06:00:00.000Z');
date.setDate(date.getDate() + i);
runs.push({
jobId: `job-limit-${i}`,
trigger: 'scheduled',
status: 'success',
startedAt: date.toISOString(),
completedAt: date.toISOString(),
duration: 10000,
unitsProcessed: 50,
pricesInserted: 50,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: date
});
}
await db.collection('scraper_runs').insertMany(runs);
const res = await request(app)
.get('/api/admin/scraper/history?limit=5')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(5);
});
it('should cap limit at 100 even if higher value requested', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=200')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
// Should not error - just caps at 100
expect(res.body).toHaveProperty('data');
expect(Array.isArray(res.body.data)).toBe(true);
});
it('should apply offset to skip records', async () => {
// Insert 5 records in order
const runs = [];
for (let i = 0; i < 5; i++) {
const date = new Date('2026-02-01T06:00:00.000Z');
date.setDate(date.getDate() + i);
runs.push({
jobId: `offset-job-${i}`,
trigger: 'scheduled',
status: 'success',
startedAt: date.toISOString(),
completedAt: date.toISOString(),
duration: 10000,
unitsProcessed: 50,
pricesInserted: 50,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: date
});
}
await db.collection('scraper_runs').insertMany(runs);
// Skip first 2 (newest two), get remaining 3
const res = await request(app)
.get('/api/admin/scraper/history?offset=2&limit=10')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(3);
// The 3rd newest should be first (offset skips the 2 newest)
expect(res.body.data[0].jobId).toBe('offset-job-2');
});
it('should return empty array when offset exceeds total records', async () => {
await db.collection('scraper_runs').insertOne({
jobId: 'single-job',
trigger: 'manual',
status: 'success',
startedAt: '2026-02-05T06:00:00.000Z',
completedAt: '2026-02-05T06:00:10.000Z',
duration: 10000,
unitsProcessed: 50,
pricesInserted: 50,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: new Date()
});
const res = await request(app)
.get('/api/admin/scraper/history?offset=100')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toEqual([]);
});
it('should accept offset of 0 as valid', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?offset=0')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body).toHaveProperty('data');
expect(Array.isArray(res.body.data)).toBe(true);
});
it('should accept limit of 1 as valid', async () => {
// Insert 3 records
const runs = [];
for (let i = 0; i < 3; i++) {
const date = new Date('2026-02-01T06:00:00.000Z');
date.setDate(date.getDate() + i);
runs.push({
jobId: `limit1-job-${i}`,
trigger: 'scheduled',
status: 'success',
startedAt: date.toISOString(),
completedAt: date.toISOString(),
duration: 10000,
unitsProcessed: 50,
pricesInserted: 50,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: [],
recordedAt: date
});
}
await db.collection('scraper_runs').insertMany(runs);
const res = await request(app)
.get('/api/admin/scraper/history?limit=1')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body.data).toHaveLength(1);
});
it('should accept limit of 100 as valid', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=100')
.set('Cookie', [`auth_token=${token}`])
.expect(200);
expect(res.body).toHaveProperty('data');
expect(Array.isArray(res.body.data)).toBe(true);
});
});
describe('Validation errors', () => {
let admin;
let token;
beforeEach(async () => {
admin = createTestAdmin();
await insertTestUser(db, admin);
token = generateTestToken(admin._id);
});
it('should return 400 for limit less than 1', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=0')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/limit/i);
});
it('should return 400 for negative limit', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=-5')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/limit/i);
});
it('should return 400 for non-numeric limit', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=abc')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/limit/i);
});
it('should return 400 for negative offset', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?offset=-1')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/offset/i);
});
it('should return 400 for non-numeric offset', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?offset=xyz')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/offset/i);
});
it('should return specific error message for invalid limit', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?limit=0')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body.error).toBe('Limit must be between 1 and 100');
});
it('should return specific error message for invalid offset', async () => {
const res = await request(app)
.get('/api/admin/scraper/history?offset=-1')
.set('Cookie', [`auth_token=${token}`])
.expect(400);
expect(res.body.error).toBe('Invalid offset parameter');
});
});
describe('Error handling', () => {
let admin;
let token;
beforeEach(async () => {
admin = createTestAdmin();
await insertTestUser(db, admin);
token = generateTestToken(admin._id);
});
it('should return 503 on database error', async () => {
const express = require('express');
const cookieParser = require('cookie-parser');
const brokenApp = express();
brokenApp.use(express.json());
brokenApp.use(cookieParser());
// Proxy db: real db for users, broken for scraper_runs
const proxyDb = {
collection: jest.fn((name) => {
if (name === 'scraper_runs') {
return {
find: jest.fn().mockReturnValue({
sort: jest.fn().mockReturnValue({
skip: jest.fn().mockReturnValue({
limit: jest.fn().mockReturnValue({
toArray: jest.fn().mockRejectedValue(new Error('Database connection lost'))
})
})
})
}),
findOne: jest.fn().mockRejectedValue(new Error('Database connection lost'))
};
}
return db.collection(name);
})
};
brokenApp.locals.db = proxyDb;
const adminRoutes = require('../../routes/admin');
brokenApp.use('/api/admin', adminRoutes);
const res = await request(brokenApp)
.get('/api/admin/scraper/history')
.set('Cookie', [`auth_token=${token}`])
.expect(503);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Service temporarily unavailable');
});
});
});
// ============================================================
// Authentication & Authorization Middleware Tests (SCRAPE-21)
// Comprehensive tests for requireAuth + requireAdmin on all
// /admin/scraper/* routes
// ============================================================
describe('Authentication & Authorization Middleware', () => {
const scraperRoutes = [
{ method: 'post', path: '/api/admin/scraper/run', expectedAdminStatus: 202 },
{ method: 'get', path: '/api/admin/scraper/status', expectedAdminStatus: 200 },
{ method: 'get', path: '/api/admin/scraper/history', expectedAdminStatus: 200 }
];
// ============================================================
// 401 - Missing JWT token
// ============================================================
describe('missing JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} without auth token`, async () => {
const res = await request(app)[method](path)
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 401 - Invalid JWT token
// ============================================================
describe('invalid JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with invalid token`, async () => {
const res = await request(app)[method](path)
.set('Cookie', ['auth_token=this-is-not-a-valid-jwt-token'])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Expired JWT token
// ============================================================
describe('expired JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with expired token`, async () => {
const adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
// Create a token that expired 1 hour ago
const secret = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
const expiredToken = jwt.sign(
{ userId: adminUser._id.toString() },
secret,
{ expiresIn: '-1h' }
);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${expiredToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Token signed with wrong secret
// ============================================================
describe('token signed with wrong secret (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with wrong-secret token`, async () => {
const adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
// Sign token with a different secret
const wrongSecretToken = jwt.sign(
{ userId: adminUser._id.toString() },
'completely-wrong-secret-key',
{ expiresIn: '7d' }
);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${wrongSecretToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Disabled user (isActive: false)
// ============================================================
describe('disabled user (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} when user is disabled`, async () => {
const disabledAdmin = createTestAdmin({ isActive: false });
await insertTestUser(db, disabledAdmin);
const token = generateTestToken(disabledAdmin._id);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 401 - Token for non-existent user
// ============================================================
describe('non-existent user (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} when user does not exist`, async () => {
const nonExistentId = new ObjectId();
const token = generateTestToken(nonExistentId);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 403 - Non-admin user
// ============================================================
describe('non-admin user (403)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 403 for ${method.toUpperCase()} ${path} for non-admin user`, async () => {
const regularUser = createTestUser({ role: 'user' });
await insertTestUser(db, regularUser);
const token = generateTestToken(regularUser._id);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(403);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Admin access required');
});
});
});
// ============================================================
// 200/202 - Valid admin user
// ============================================================
describe('valid admin user (200/202)', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
scraperRoutes.forEach(({ method, path, expectedAdminStatus }) => {
it(`should return ${expectedAdminStatus} for ${method.toUpperCase()} ${path} for admin user`, async () => {
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${adminToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(expectedAdminStatus);
expect(res.body).toHaveProperty('data');
});
});
});
// ============================================================
// Rate Limiting for POST /api/admin/scraper/run (SCRAPE-22)
// ============================================================
describe('rate limiting', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
// Reset the rate limiter between tests
const { resetRateLimiter } = require('../../routes/admin');
resetRateLimiter();
});
it('should allow the first 5 requests within the rate limit window', async () => {
for (let i = 0; i < 5; i++) {
// Reset scraper state to idle before each request so mutex does not block
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// Wait for async scrape to release lock
await new Promise(resolve => setTimeout(resolve, 50));
}
});
it('should return 429 when the 6th request exceeds the rate limit', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request should be rate limited
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/rate limit/i);
});
it('should include Retry-After header in 429 response', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.headers).toHaveProperty('retry-after');
const retryAfter = parseInt(res.headers['retry-after']);
expect(retryAfter).toBeGreaterThan(0);
// Should be less than or equal to 3600 seconds (1 hour)
expect(retryAfter).toBeLessThanOrEqual(3600);
});
it('should track rate limits per user (different admins have separate limits)', async () => {
// Create a second admin
const admin2 = createTestAdmin();
await insertTestUser(db, admin2);
const admin2Token = generateTestToken(admin2._id);
// Use up first admin's rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify first admin is rate limited
scraperJob.__reset();
const res1 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res1.status).toBe(429);
// Second admin should still be able to make requests
scraperJob.__reset();
const res2 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${admin2Token}`])
.send({});
expect(res2.status).toBe(202);
});
it('should reset rate limit after window expires', async () => {
// Access the rate limiter internals for testing
const adminModule = require('../../routes/admin');
// Use up the rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify rate limited
scraperJob.__reset();
const blockedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(blockedRes.status).toBe(429);
// Simulate window expiry by resetting the rate limiter
adminModule.resetRateLimiter();
// Should be allowed again after window reset
scraperJob.__reset();
const allowedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(allowedRes.status).toBe(202);
});
it('should check rate limit before mutex (rate limit takes precedence)', async () => {
// Use up rate limit
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Set scraper as running (would normally get 409)
scraperJob._setState(true, 'existing-job');
// Should get 429 (rate limit), not 409 (mutex conflict)
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
});
});
// ============================================================
// Middleware ordering: requireAuth runs before requireAdmin
// ============================================================
describe('middleware ordering', () => {
it('should return 401 (not 403) when token is missing, even for non-admin scenario', async () => {
// Without any token, requireAuth should reject with 401
// before requireAdmin ever runs
const res = await request(app)
.get('/api/admin/scraper/status');
expect(res.status).toBe(401);
expect(res.body.error).toBe('Authentication required');
});
it('should return 401 (not 403) for disabled admin user', async () => {
// A disabled admin should get 401 from requireAuth
// even though they have the admin role
const disabledAdmin = createTestAdmin({ isActive: false, role: 'admin' });
await insertTestUser(db, disabledAdmin);
const token = generateTestToken(disabledAdmin._id);
const res = await request(app)
.get('/api/admin/scraper/status')
.set('Cookie', [`auth_token=${token}`]);
expect(res.status).toBe(401);
expect(res.body.error).toBe('Authentication required');
});
});
});
}); });

296
__tests__/scraper/scraperService.test.js
View File

@ -20,10 +20,9 @@ let mongoServer;
let client; let client;
let db; let db;
// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation // We will require runScrape and recordScraperRun after implementation
let runScrape; let runScrape;
let recordScraperRun; let recordScraperRun;
let createScraperIndexes;
// Store original module references so we can mock individual functions // Store original module references so we can mock individual functions
let scraperService; let scraperService;
@ -37,7 +36,7 @@ beforeAll(async () => {
// Dynamically require to pick up implementation // Dynamically require to pick up implementation
scraperService = require('../../services/scraperService'); scraperService = require('../../services/scraperService');
({ runScrape, recordScraperRun, createScraperIndexes } = scraperService); ({ runScrape, recordScraperRun } = scraperService);
}); });
afterAll(async () => { afterAll(async () => {
@ -220,11 +219,11 @@ describe('runScrape', () => {
expect(result.staleUnitsCount).toBe(0); expect(result.staleUnitsCount).toBe(0);
// Verify no units were written to the units collection // Verify no units were written to the units collection
const unitsCount = await db.collection('units_scraper').countDocuments(); const unitsCount = await db.collection('units_migration_test').countDocuments();
expect(unitsCount).toBe(0); expect(unitsCount).toBe(0);
// Verify no prices were written // Verify no prices were written
const pricesCount = await db.collection('unit_prices_scraper').countDocuments(); const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
expect(pricesCount).toBe(0); expect(pricesCount).toBe(0);
}); });
@ -278,7 +277,7 @@ describe('runScrape', () => {
const faultyDb = { const faultyDb = {
collection: (name) => { collection: (name) => {
const realCollection = db.collection(name); const realCollection = db.collection(name);
if (name === 'units_scraper') { if (name === 'units_migration_test') {
return new Proxy(realCollection, { return new Proxy(realCollection, {
get(target, prop) { get(target, prop) {
if (prop === 'bulkWrite') { if (prop === 'bulkWrite') {
@ -344,7 +343,7 @@ describe('runScrape', () => {
const faultyDb = { const faultyDb = {
collection: (name) => { collection: (name) => {
const realCollection = db.collection(name); const realCollection = db.collection(name);
if (name === 'unit_prices_scraper') { if (name === 'unit_prices_migration_test') {
return new Proxy(realCollection, { return new Proxy(realCollection, {
get(target, prop) { get(target, prop) {
if (prop === 'bulkWrite') { if (prop === 'bulkWrite') {
@ -389,7 +388,7 @@ describe('runScrape', () => {
const yesterdayStr = yesterday.toISOString().split('T')[0]; const yesterdayStr = yesterday.toISOString().split('T')[0];
// Yesterday had units: OLD-A, OLD-B, OLD-C // Yesterday had units: OLD-A, OLD-B, OLD-C
await db.collection('unit_prices_scraper').insertMany([ await db.collection('unit_prices_migration_test').insertMany([
{ unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 }, { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
{ unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 }, { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
{ unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 } { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
@ -456,284 +455,3 @@ describe('runScrape', () => {
expect(result.errors).toContain('No units found in HTML'); expect(result.errors).toContain('No units found in HTML');
}); });
}); });
// ============================================================
// Test: sanitizeError - Error message sanitization
// ============================================================
describe('sanitizeError', () => {
let sanitizeError;
beforeAll(() => {
({ sanitizeError } = require('../../services/scraperService'));
});
// ----------------------------------------------------------
// File path sanitization
// ----------------------------------------------------------
describe('file path sanitization', () => {
it('should remove Unix absolute file paths from error messages', () => {
const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/home\/user/);
expect(sanitized.message).toContain('ENOENT');
});
it('should remove /var paths from error messages', () => {
const error = new Error('Failed to read /var/app/current/data/secrets.yml');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/var\/app/);
expect(sanitized.message).toContain('Failed to read');
});
it('should remove Windows-style file paths from error messages', () => {
const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/C:\\Users/);
expect(sanitized.message).toContain('Cannot find module');
});
it('should remove /tmp and /usr paths from error messages', () => {
const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/tmp\//);
expect(sanitized.message).not.toMatch(/\/usr\//);
});
});
// ----------------------------------------------------------
// MongoDB connection string sanitization
// ----------------------------------------------------------
describe('connection string sanitization', () => {
it('should redact mongodb:// connection strings', () => {
const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('s3cretP4ss');
expect(sanitized.message).not.toContain('admin:');
expect(sanitized.message).toContain('Connection failed');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact mongodb+srv:// connection strings', () => {
const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('password123');
expect(sanitized.message).not.toContain('user:');
expect(sanitized.message).toContain('Timeout connecting to');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact connection string without credentials', () => {
const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
});
// ----------------------------------------------------------
// Credential / secret sanitization
// ----------------------------------------------------------
describe('credential sanitization', () => {
it('should redact common environment variable patterns', () => {
const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('sk-abc123xyz');
expect(sanitized.message).not.toContain('bearer-9876');
});
it('should redact password patterns', () => {
const error = new Error('Auth failed with password=MyS3cret!');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('MyS3cret!');
});
});
// ----------------------------------------------------------
// Error type preservation
// ----------------------------------------------------------
describe('error type preservation', () => {
it('should preserve the error type/name', () => {
const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TypeError');
});
it('should preserve custom error names', () => {
const error = new Error('Timeout at /var/app/scraper.js:100');
error.name = 'TimeoutError';
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TimeoutError');
});
it('should preserve error name for RangeError', () => {
const error = new RangeError('Maximum call stack size exceeded');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('RangeError');
});
});
// ----------------------------------------------------------
// General description preserved for debugging
// ----------------------------------------------------------
describe('general description preservation', () => {
it('should preserve a useful general description', () => {
const error = new Error('Network timeout after 30000ms');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('Network timeout after 30000ms');
});
it('should preserve error description when no sensitive data present', () => {
const error = new Error('Request failed with status code 500');
const sanitized = sanitizeError(error);
expect(sanitized.message).toBe('Request failed with status code 500');
});
it('should return a useful message even after heavy sanitization', () => {
const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('ECONNREFUSED');
expect(sanitized.message.length).toBeGreaterThan(5);
});
});
// ----------------------------------------------------------
// Stack trace sanitization
// ----------------------------------------------------------
describe('stack trace removal', () => {
it('should remove file paths from stack traces', () => {
const error = new Error('Something failed');
error.stack = 'Error: Something failed\n at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
const sanitized = sanitizeError(error);
expect(sanitized.stack).not.toMatch(/\/home\/user/);
expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
});
it('should handle errors without stack trace', () => {
const error = new Error('No stack');
error.stack = undefined;
const sanitized = sanitizeError(error);
expect(sanitized.stack).toBeUndefined();
});
});
// ----------------------------------------------------------
// Integration: sanitization applied before recordScraperRun()
// ----------------------------------------------------------
describe('sanitization before recordScraperRun()', () => {
it('should store sanitized error message in scraper_runs on failure', async () => {
const html = createSampleHtml(['SANITIZE-A']);
// Create a db proxy that throws an error containing sensitive info
const sensitiveError = new Error(
'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
);
sensitiveError.name = 'MongoServerError';
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'units_scraper') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
return async () => { throw sensitiveError; };
}
const value = target[prop];
if (typeof value === 'function') {
return value.bind(target);
}
return value;
}
});
}
return realCollection;
}
};
const result = await runScrape(faultyDb, {
jobId: 'test-sanitized-error',
htmlContent: html
});
expect(result.status).toBe('failed');
// Verify the error stored in result.errors is sanitized
const errorMsg = result.errors[0];
expect(errorMsg).not.toContain('SuperSecret');
expect(errorMsg).not.toContain('admin:');
expect(errorMsg).not.toContain('/home/deploy/');
expect(errorMsg).toContain('MongoServerError');
// Verify the error stored in scraper_runs is sanitized
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('failed');
const storedError = runRecord.errors[0];
expect(storedError).not.toContain('SuperSecret');
expect(storedError).not.toContain('admin:');
expect(storedError).not.toContain('/home/deploy/');
});
});
});
// ============================================================
// Test: createScraperIndexes
// ============================================================
describe('createScraperIndexes', () => {
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
beforeEach(() => {
mockLogger.info.mockClear();
mockLogger.warn.mockClear();
mockLogger.error.mockClear();
});
it('should create the status_startedAt compound index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
expect(statusIndex).toBeTruthy();
expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
});
it('should create the startedAt_desc index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(startedAtIndex).toBeTruthy();
expect(startedAtIndex.key).toEqual({ startedAt: -1 });
});
it('should log success after creating indexes', async () => {
await createScraperIndexes(db, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
});
it('should be idempotent (no error on second call)', async () => {
await createScraperIndexes(db, mockLogger);
// Calling again should not throw
await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
// Indexes should still exist
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(statusIndex).toBeTruthy();
expect(startedAtIndex).toBeTruthy();
});
it('should handle database errors gracefully', async () => {
// Pass an object that will throw when collection() is called
const faultyDb = {
collection: () => {
throw new Error('Connection lost');
}
};
await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
});
});

177
__tests__/scraper/serverIntegration.test.js
View File

@ -1,177 +0,0 @@
/**
* Tests for scraper integration in server.js startup
*
* Covers:
* - Server initializes scraper scheduler on startup
* - Server does not crash if scheduler init fails
* - Indexes are created before scheduler starts
* - Scheduler not initialized when SCRAPER_ENABLED=false
*
* Strategy: Instead of importing server.js directly (which starts Express and
* connects to MongoDB), we replicate the scraper initialization logic from
* connectToMongoDB() and test it with mocked dependencies. This verifies the
* integration contract without needing a full server boot.
*/
const { MongoClient } = require('mongodb');
describe('Server Scraper Integration', () => {
let connection;
let db;
beforeAll(async () => {
const uri = process.env.MONGO_URI;
connection = await MongoClient.connect(uri);
db = connection.db('apartments_server_integration_test');
});
afterAll(async () => {
if (connection) {
await connection.close();
}
});
describe('scraper initialization during startup', () => {
let mockCreateScraperIndexes;
let mockInitializeScheduler;
let mockCreateLogger;
let mockLogger;
beforeEach(() => {
mockLogger = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
debug: jest.fn()
};
mockCreateLogger = jest.fn(() => mockLogger);
mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
mockInitializeScheduler = jest.fn();
});
/**
* Replicates the scraper initialization logic from server.js connectToMongoDB().
* This is the exact pattern used in production:
*
* try {
* const scraperLogger = createLogger('server-init');
* await createScraperIndexes(db, scraperLogger);
* if (scraperConfig.SCRAPER_ENABLED) {
* initializeScheduler(db);
* }
* } catch (error) {
* console.error('Scraper initialization failed (server continues):', error.message);
* }
*/
async function runScraperInit(testDb, config) {
try {
const scraperLogger = mockCreateLogger('server-init');
await mockCreateScraperIndexes(testDb, scraperLogger);
if (config.SCRAPER_ENABLED) {
mockInitializeScheduler(testDb);
}
return { success: true };
} catch (error) {
return { success: false, error: error.message };
}
}
it('should initialize scraper scheduler on startup when enabled', async () => {
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
});
it('should not crash if createScraperIndexes throws an error', async () => {
mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
// The server should continue (error is caught, not thrown)
expect(result.success).toBe(false);
expect(result.error).toBe('Index creation failed');
// initializeScheduler should NOT have been called since the error occurred before it
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
it('should create indexes before initializing the scheduler', async () => {
const callOrder = [];
mockCreateScraperIndexes.mockImplementation(async () => {
callOrder.push('createScraperIndexes');
});
mockInitializeScheduler.mockImplementation(() => {
callOrder.push('initializeScheduler');
});
const config = { SCRAPER_ENABLED: true };
await runScraperInit(db, config);
expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
});
it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
const config = { SCRAPER_ENABLED: false };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
});
describe('server.js imports and exports verification', () => {
it('should be able to import createScraperIndexes from scraperService', () => {
const { createScraperIndexes } = require('../../services/scraperService');
expect(typeof createScraperIndexes).toBe('function');
});
it('should be able to import initializeScheduler from scraperJob', () => {
const { initializeScheduler } = require('../../jobs/scraperJob');
expect(typeof initializeScheduler).toBe('function');
});
it('should be able to import registerSignalHandlers from scraperJob', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
expect(typeof registerSignalHandlers).toBe('function');
});
it('should be able to import SCRAPER_ENABLED from scraper config', () => {
const scraperConfig = require('../../config/scraper');
expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
});
});
describe('createScraperIndexes error handling with real db', () => {
it('should handle index creation gracefully with a valid db', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const { createLogger } = require('../../services/scraperLogger');
const logger = createLogger('test-server-init');
// Should not throw - creates indexes on the test db
await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
});
it('should throw when given an invalid db object', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
// Passing null should throw (simulates what the try/catch in server.js protects against)
await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
});
});
describe('registerSignalHandlers', () => {
it('should register SIGTERM and SIGINT handlers without throwing', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
// Should not throw when called
expect(() => registerSignalHandlers()).not.toThrow();
});
});
});

2
__tests__/scraper/upsertUnits.test.js
View File

@ -56,7 +56,7 @@ beforeEach(async () => {
}); });
describe('upsertUnits', () => { describe('upsertUnits', () => {
const UNITS_COLLECTION = 'units_scraper'; const UNITS_COLLECTION = 'units_migration_test';
// --------------------------------------------------------------- // ---------------------------------------------------------------
// 1. bulkWrite is called with correct updateOne operations // 1. bulkWrite is called with correct updateOne operations

9
config/scraper.js
View File

@ -13,7 +13,7 @@ module.exports = {
// Scheduling configuration // Scheduling configuration
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *', SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver', SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false', SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings // HTTP settings
@ -30,11 +30,10 @@ module.exports = {
// Graceful shutdown timeout (how long to wait for running job before force-stopping) // Graceful shutdown timeout (how long to wait for running job before force-stopping)
SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000, SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
// MongoDB collection names - defaults to validation collections for scraper validation; // MongoDB collection names (environment variable overrides for development isolation)
// switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
COLLECTIONS: { COLLECTIONS: {
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper', UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper', PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries', DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs' SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
} }

129
routes/admin.js
View File

@ -1156,59 +1156,8 @@ const {
getNextScheduledRun getNextScheduledRun
} = require('../jobs/scraperJob'); } = require('../jobs/scraperJob');
const { runScrape } = require('../services/scraperService'); const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger');
const scraperConfig = require('../config/scraper'); const scraperConfig = require('../config/scraper');
// Logger for scraper admin routes
const scraperRouteLogger = createLogger('scraper-admin');
// Rate limiting for manual scrape trigger (per-user, in-memory)
const RATE_LIMIT_MAX_REQUESTS = 5;
const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
const rateLimitStore = new Map();
/**
* Check rate limit for a given user ID.
* Returns an object indicating whether the request is allowed.
*
* @param {string} userId - The user ID to check
* @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
*/
function checkRateLimit(userId) {
const now = Date.now();
const userKey = userId.toString();
if (!rateLimitStore.has(userKey)) {
rateLimitStore.set(userKey, []);
}
const timestamps = rateLimitStore.get(userKey);
// Remove timestamps outside the current window
const windowStart = now - RATE_LIMIT_WINDOW_MS;
const validTimestamps = timestamps.filter(ts => ts > windowStart);
rateLimitStore.set(userKey, validTimestamps);
if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
// Calculate when the oldest request in the window will expire
const oldestTimestamp = validTimestamps[0];
const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
return { allowed: false, retryAfterSeconds };
}
// Record this request
validTimestamps.push(now);
return { allowed: true, retryAfterSeconds: null };
}
/**
* Reset the rate limiter (for testing)
*/
function resetRateLimiter() {
rateLimitStore.clear();
}
/** /**
* POST /api/admin/scraper/run * POST /api/admin/scraper/run
* Trigger a manual scrape * Trigger a manual scrape
@ -1222,15 +1171,6 @@ router.post('/scraper/run', async (req, res) => {
const db = req.app.locals.db; const db = req.app.locals.db;
const { dryRun = false, htmlContent = null } = req.body || {}; const { dryRun = false, htmlContent = null } = req.body || {};
// Check rate limit (per-user, before mutex check)
const rateLimitResult = checkRateLimit(req.user._id);
if (!rateLimitResult.allowed) {
res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
return res.status(429).json({
error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
});
}
// Check if scraper is already running // Check if scraper is already running
if (isScraperRunning()) { if (isScraperRunning()) {
return res.status(409).json({ error: 'Scrape already in progress' }); return res.status(409).json({ error: 'Scrape already in progress' });
@ -1250,10 +1190,7 @@ router.post('/scraper/run', async (req, res) => {
// Start scrape asynchronously (do not await - return 202 immediately) // Start scrape asynchronously (do not await - return 202 immediately)
runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent }) runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
.catch((error) => { .catch((error) => {
scraperRouteLogger.error('Async scrape failed', { console.error('Async scrape failed:', error.message);
errorType: error.name,
errorMessage: error.message
});
}) })
.finally(() => releaseLock()); .finally(() => releaseLock());
@ -1268,10 +1205,7 @@ router.post('/scraper/run', async (req, res) => {
}); });
} catch (error) { } catch (error) {
scraperRouteLogger.error('Error triggering scrape', { console.error('Error triggering scrape:', error);
errorType: error.name,
errorMessage: error.message
});
res.status(500).json({ error: 'Failed to start scrape job' }); res.status(500).json({ error: 'Failed to start scrape job' });
} }
}); });
@ -1311,67 +1245,10 @@ router.get('/scraper/status', async (req, res) => {
}); });
} catch (error) { } catch (error) {
scraperRouteLogger.error('Error fetching scraper status', { console.error('Error fetching scraper status:', error);
errorType: error.name,
errorMessage: error.message
});
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
/**
* GET /api/admin/scraper/history
* Get scraper run history with pagination
*
* Query params:
* - limit: Number of records (1-100, default 30)
* - offset: Number of records to skip (default 0)
*/
router.get('/scraper/history', async (req, res) => {
try {
const db = req.app.locals.db;
// Parse and validate pagination parameters
let limit = parseInt(req.query.limit);
let offset = parseInt(req.query.offset);
// Validate limit
if (req.query.limit !== undefined) {
if (isNaN(limit) || limit < 1) {
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
}
limit = Math.min(limit, 100);
} else {
limit = 30;
}
// Validate offset
if (req.query.offset !== undefined) {
if (isNaN(offset) || offset < 0) {
return res.status(400).json({ error: 'Invalid offset parameter' });
}
} else {
offset = 0;
}
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.find({})
.sort({ startedAt: -1 })
.skip(offset)
.limit(limit)
.toArray();
res.json({ data: history });
} catch (error) {
scraperRouteLogger.error('Error fetching scraper history', {
errorType: error.name,
errorMessage: error.message
});
res.status(503).json({ error: 'Service temporarily unavailable' }); res.status(503).json({ error: 'Service temporarily unavailable' });
} }
}); });
module.exports = router; module.exports = router;
module.exports.clearStatsCache = clearStatsCache; module.exports.clearStatsCache = clearStatsCache;
module.exports.resetRateLimiter = resetRateLimiter;

35
server.js
View File

@ -11,9 +11,6 @@ const activityRoutes = require('./routes/activity');
const adminRoutes = require('./routes/admin'); const adminRoutes = require('./routes/admin');
const { createIndexes } = require('./models/user'); const { createIndexes } = require('./models/user');
const { createActivityIndexes } = require('./services/activityLogger'); const { createActivityIndexes } = require('./services/activityLogger');
const { createScraperIndexes } = require('./services/scraperService');
const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
const scraperConfig = require('./config/scraper');
const app = express(); const app = express();
const PORT = process.env.PORT || 3000; const PORT = process.env.PORT || 3000;
@ -83,32 +80,14 @@ async function connectToMongoDB() {
await createIndexes(db); await createIndexes(db);
await createActivityIndexes(db); await createActivityIndexes(db);
console.log('✅ Passport configured and indexes created'); console.log('✅ Passport configured and indexes created');
// Initialize scraper (indexes + scheduler) - errors logged but don't crash server
try {
const { createLogger } = require('./services/scraperLogger');
const scraperLogger = createLogger('server-init');
await createScraperIndexes(db, scraperLogger);
if (scraperConfig.SCRAPER_ENABLED) {
initializeScheduler(db);
console.log('✅ Scraper scheduler initialized');
} else {
console.log('ℹ️ Scraper scheduling disabled');
}
} catch (error) {
console.error('⚠️ Scraper initialization failed (server continues):', error.message);
}
} catch (error) { } catch (error) {
console.error('❌ Failed to connect to MongoDB:', error); console.error('❌ Failed to connect to MongoDB:', error);
process.exit(1); process.exit(1);
} }
} }
// Register scraper signal handlers for graceful shutdown // Helper function to get today's date in UTC
registerSignalHandlers(); const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
// Helper function to get today's date in configured timezone
const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
// Cache for the most recent date with data // Cache for the most recent date with data
let cachedLatestDate = null; let cachedLatestDate = null;
@ -143,7 +122,7 @@ const getLatestDateWithData = async () => {
} }
// Fallback to UTC today if no data found // Fallback to UTC today if no data found
return getTodayDate(); return getTodayDateUTC();
}; };
// Helper function to get yesterday relative to the latest date with data // Helper function to get yesterday relative to the latest date with data
@ -641,10 +620,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
]).toArray(); ]).toArray();
// 4. Historical Comparison - current vs 30/90/365 day averages // 4. Historical Comparison - current vs 30/90/365 day averages
const tz = scraperConfig.SCRAPER_TIMEZONE; const now = new Date();
const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const [currentAvg, avg30, avg90, avg365] = await Promise.all([ const [currentAvg, avg30, avg90, avg365] = await Promise.all([
db.collection(PRICES_COLLECTION).aggregate([ db.collection(PRICES_COLLECTION).aggregate([

100
services/scraperService.js
View File

@ -626,11 +626,11 @@ async function updateDailySummary(db, summaryData, logger) {
// ============================================================ // ============================================================
/** /**
* Get today's date in YYYY-MM-DD format using the configured timezone. * Get today's date in YYYY-MM-DD format (UTC).
* @returns {string} Today's date string * @returns {string} Today's date string
*/ */
function getToday() { function getTodayUTC() {
return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE }); return new Date().toISOString().split('T')[0];
} }
/** /**
@ -651,54 +651,6 @@ async function getYesterdayUnitCodes(db, today) {
return new Set(yesterdayRecords.map(r => r.unit_code)); return new Set(yesterdayRecords.map(r => r.unit_code));
} }
// ============================================================
// Error Sanitization
// ============================================================
/**
* Sanitize an error object to remove sensitive information before storage.
* Removes file paths, connection strings, and credential patterns while
* preserving the error type and a useful general description for debugging.
*
* @param {Error} error - Error object to sanitize
* @returns {Object} Sanitized error with name, message, and optionally stack
*/
function sanitizeError(error) {
const sanitized = {
name: error.name || 'Error',
message: sanitizeMessage(error.message || ''),
};
if (error.stack) {
sanitized.stack = sanitizeMessage(error.stack);
}
return sanitized;
}
/**
* Sanitize a string message by removing sensitive patterns.
* @param {string} message - Raw error message
* @returns {string} Sanitized message
*/
function sanitizeMessage(message) {
let result = message;
// Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
// Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
// Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
// Remove Windows-style absolute paths (C:\..., D:\...)
result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
return result;
}
// ============================================================ // ============================================================
// Scraper Run History // Scraper Run History
// ============================================================ // ============================================================
@ -730,37 +682,6 @@ async function recordScraperRun(db, runData, logger) {
} }
} }
// ============================================================
// Index Management
// ============================================================
/**
* Create indexes on the scraper_runs collection.
* Should be called once during application startup.
* Idempotent - safe to call multiple times.
*
* @param {Db} db - MongoDB database instance
* @param {Object} logger - Logger instance
* @returns {Promise<void>}
*/
async function createScraperIndexes(db, logger) {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
// Compound index for querying runs by status sorted by most recent
await collection.createIndex(
{ status: 1, startedAt: -1 },
{ name: 'status_startedAt' }
);
// Index for sorting all runs by start time (most recent first)
await collection.createIndex(
{ startedAt: -1 },
{ name: 'startedAt_desc' }
);
logger.info('Scraper indexes created successfully');
}
// ============================================================ // ============================================================
// Main Orchestration Function // Main Orchestration Function
// ============================================================ // ============================================================
@ -820,7 +741,7 @@ async function runScrape(db, options = {}) {
result.unitsProcessed = units.length; result.unitsProcessed = units.length;
// Step 4: Database operations // Step 4: Database operations
const today = getToday(); const today = getTodayUTC();
// Get yesterday's unit codes for comparison // Get yesterday's unit codes for comparison
const yesterdayUnits = await getYesterdayUnitCodes(db, today); const yesterdayUnits = await getYesterdayUnitCodes(db, today);
@ -866,13 +787,12 @@ async function runScrape(db, options = {}) {
result.status = 'success'; result.status = 'success';
} catch (error) { } catch (error) {
const cleanError = sanitizeError(error);
logger.error('Scrape failed', { logger.error('Scrape failed', {
errorType: cleanError.name, errorType: error.name,
errorMessage: cleanError.message errorMessage: error.message
}); });
result.status = 'failed'; result.status = 'failed';
result.errors.push(cleanError.message); result.errors.push(error.message);
} finally { } finally {
result.completedAt = new Date().toISOString(); result.completedAt = new Date().toISOString();
@ -902,9 +822,8 @@ module.exports = {
markStaleUnits, markStaleUnits,
updateDailySummary, updateDailySummary,
recordScraperRun, recordScraperRun,
createScraperIndexes,
// Export helpers for testing // Export helpers for testing
getToday, getTodayUTC,
getYesterdayUnitCodes, getYesterdayUnitCodes,
getYesterday, getYesterday,
parseInteger, parseInteger,
@ -915,6 +834,5 @@ module.exports = {
parseFloatValue, parseFloatValue,
trimString, trimString,
isRetryableError, isRetryableError,
sleep, sleep
sanitizeError
}; };