stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

Compare commits

base: stephen:dev
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field
..
compare: stephen:ef15beb622aa51bc9d8b32a0a66f828f435b56fa
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field

1 Commits
dev ... ef15beb622

Author SHA1 Message Date
Stephen Minakian
ef15beb622 Add GET /api/admin/scraper/history endpoint with pagination
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
Details
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 42s
Details
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
Details
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
Details
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Details 
Implement the scraper history endpoint that returns past scrape run
records from the scraper_runs collection. Features include:

- GET /api/admin/scraper/history protected by requireAuth + requireAdmin
- Pagination via limit (1-100, default 30) and offset (default 0) params
- Input validation returning 400 for invalid limit/offset values
- Results sorted by startedAt descending (newest first)
- Graceful error handling returning 503 on database failures
- 20 new tests covering auth, pagination, validation, and error cases
 2026-02-06 22:33:01 -07:00
16 changed files with 261 additions and 1680 deletions
Expand all files Collapse all files

25
.github/workflows/deploy.yml vendored
View File

@ -44,10 +44,8 @@ jobs:
set +e
OUTPUT=$(npm run lint 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "lint_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
@ -58,10 +56,8 @@ jobs:
set +e
OUTPUT=$(npm test -- --runInBand 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "test_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
env:
@ -102,9 +98,9 @@ jobs:
OVERALL_STATUS="failure"
fi
# Outputs are already truncated at the source (ci job)
LINT_OUTPUT="$RAW_LINT_OUTPUT"
TEST_OUTPUT="$RAW_TEST_OUTPUT"
# Truncate outputs if too long (max 10000 chars each)
LINT_OUTPUT="${RAW_LINT_OUTPUT:0:10000}"
TEST_OUTPUT="${RAW_TEST_OUTPUT:0:10000}"
# Build JSON payload
PAYLOAD=$(jq -n \
@ -264,13 +260,12 @@ jobs:
# ============================================================
# SBOM Generation with Syft
# ============================================================
- name: Install Syft
run: |
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
- name: Generate SBOM with Syft
run: |
syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json
uses: anchore/sbom-action@v0
with:
image: ${{ steps.set-tag.outputs.full_image }}
format: spdx-json
output-file: sbom.spdx.json
# ============================================================
# Image Signing with Cosign

64
__tests__/scraper/fixtures/sample-listing-call.html
View File

@ -1,64 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - Call for Pricing</title></head>
<body>
<section class="spaces__tab-unit">
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9001"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="750"
data-spaces-sort-bed="1"
data-spaces-unit="P-101"
data-spaces-unit-id="9001"
data-spaces-unit-floor="1"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="500"
data-spaces-sort-plan-name="Penthouse A"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=9001"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
aria-label="Unit P-101">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-101</span>
</div>
</article>
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9002"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="900"
data-spaces-sort-bed="2"
data-spaces-unit="P-102"
data-spaces-unit-id="9002"
data-spaces-unit-floor="2"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="501"
data-spaces-sort-plan-name="Penthouse B"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=9002"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
aria-label="Unit P-102">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-102</span>
</div>
</article>
</section>
</body>
</html>

9
__tests__/scraper/fixtures/sample-listing-empty.html
View File

@ -1,9 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - No Units</title></head>
<body>
<section class="spaces__tab-unit">
<!-- Empty - no units available -->
</section>
</body>
</html>

426
__tests__/scraper/fixtures/sample-listing.html
View File

@ -1,288 +1,144 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - Apartments</title></head>
<html lang="en">
<head>
<title>Country Club Towers - Available Units</title>
</head>
<body>
<section class="spaces__tab-unit">
<article
class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154842"
data-spaces-soonest="2025-10-11"
data-spaces-sort-date="1760140800"
data-spaces-sort-price="5230"
data-spaces-sort-area="1210"
data-spaces-sort-bed="2"
data-spaces-unit="W2707"
data-spaces-unit-id="154842"
data-spaces-unit-floor="2755"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="10270"
data-spaces-sort-plan-name="Pyramid Peak - Terrace"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154842"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
aria-label="Unit W2707">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">5230</span>
<span class="spaces-unit__unit">W2707</span>
</div>
</article>
<article
class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154451"
data-spaces-soonest="2026-03-08"
data-spaces-sort-date="1772928000"
data-spaces-sort-price="2767"
data-spaces-sort-area="806"
data-spaces-sort-bed="1"
data-spaces-unit="E0901"
data-spaces-unit-id="154451"
data-spaces-unit-floor="2738"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8016"
data-spaces-sort-plan-name="Sunshine Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154451"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
aria-label="Unit E0901">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2767</span>
<span class="spaces-unit__unit">E0901</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154814"
data-spaces-soonest="2025-12-18"
data-spaces-sort-date="1766016000"
data-spaces-sort-price="2255"
data-spaces-sort-area="623"
data-spaces-sort-bed="0"
data-spaces-unit="W2603"
data-spaces-unit-id="154814"
data-spaces-unit-floor="2754"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8018"
data-spaces-sort-plan-name="Little Bear Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154814"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
aria-label="Unit W2603">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2255</span>
<span class="spaces-unit__unit">W2603</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154428"
data-spaces-soonest="2026-02-22"
data-spaces-sort-date="1771718400"
data-spaces-sort-price="4250"
data-spaces-sort-area="1152"
data-spaces-sort-bed="2"
data-spaces-unit="W0802"
data-spaces-unit-id="154428"
data-spaces-unit-floor="2737"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8017"
data-spaces-sort-plan-name="Mt. Princeton"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154428"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
aria-label="Unit W0802">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">4250</span>
<span class="spaces-unit__unit">W0802</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154778"
data-spaces-soonest="2025-12-31"
data-spaces-sort-date="1767139200"
data-spaces-sort-price="2495"
data-spaces-sort-area="764"
data-spaces-sort-bed="1"
data-spaces-unit="W2405"
data-spaces-unit-id="154778"
data-spaces-unit-floor="2752"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8020"
data-spaces-sort-plan-name="Grays Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154778"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
aria-label="Unit W2405">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W2405</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154442"
data-spaces-soonest="2025-02-01"
data-spaces-sort-date="1738368000"
data-spaces-sort-price="2495"
data-spaces-sort-area="802"
data-spaces-sort-bed="1"
data-spaces-unit="W0809"
data-spaces-unit-id="154442"
data-spaces-unit-floor="2737"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8022"
data-spaces-sort-plan-name="Pikes Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154442"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
aria-label="Unit W0809">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W0809</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154764"
data-spaces-soonest="2026-02-17"
data-spaces-sort-date="1771286400"
data-spaces-sort-price="2435"
data-spaces-sort-area="715"
data-spaces-sort-bed="1"
data-spaces-unit="W2308"
data-spaces-unit-id="154764"
data-spaces-unit-floor="2751"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8023"
data-spaces-sort-plan-name="Longs Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154764"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
aria-label="Unit W2308">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2435</span>
<span class="spaces-unit__unit">W2308</span>
</div>
</article>
<article
class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154612"
data-spaces-soonest="2026-03-24"
data-spaces-sort-date="1774310400"
data-spaces-sort-price="2683"
data-spaces-sort-area="797"
data-spaces-sort-bed="1"
data-spaces-unit="W1610"
data-spaces-unit-id="154612"
data-spaces-unit-floor="2744"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8024"
data-spaces-sort-plan-name="Torreys Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154612"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
aria-label="Unit W1610">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2683</span>
<span class="spaces-unit__unit">W1610</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154495"
data-spaces-soonest="2025-11-07"
data-spaces-sort-date="1762473600"
data-spaces-sort-price="4470"
data-spaces-sort-area="1230"
data-spaces-sort-bed="2"
data-spaces-unit="E1011"
data-spaces-unit-id="154495"
data-spaces-unit-floor="2739"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8025"
data-spaces-sort-plan-name="Maroon Peak"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154495"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
aria-label="Unit E1011">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">4470</span>
<span class="spaces-unit__unit">E1011</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="154919"
data-spaces-soonest="2025-10-27"
data-spaces-sort-date="1761523200"
data-spaces-sort-price="8581"
data-spaces-sort-area="1532"
data-spaces-sort-bed="2"
data-spaces-unit="E3205"
data-spaces-unit-id="154919"
data-spaces-unit-floor="2760"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8037"
data-spaces-sort-plan-name="Mt. Antero"
data-spaces-sort-bath="2.5"
data-spaces-href="?spaces_tab=unit-detail&detail=154919"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
aria-label="Unit E3205">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">8581</span>
<span class="spaces-unit__unit">E3205</span>
</div>
</article>
</section>
<div class="spaces-container">
<section class="spaces__tab-unit">
<!-- Unit 1: Fully populated with all attributes and an image -->
<article
data-spaces-id="1001"
data-spaces-unit="CCT-101"
data-spaces-unit-id="5001"
data-spaces-unit-floor="1"
data-spaces-sort-area="750"
data-spaces-sort-bed="1"
data-spaces-sort-bath="1"
data-spaces-sort-price="1450"
data-spaces-available="true"
data-spaces-unavailable="false"
data-spaces-soonest="2026-03-01"
data-spaces-sort-date="20260301"
data-spaces-plan-id="201"
data-spaces-sort-plan-name="Alpine"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100"
data-spaces-href="/units/CCT-101"
data-spaces-inventory-href="/inventory/CCT-101"
data-spaces-specials-content="First month free!"
>
<div class="unit-card">
<img src="https://example.com/images/unit-101.jpg" alt="Unit 101" />
<h3>Unit CCT-101</h3>
<p>1 Bed / 1 Bath - $1,450/mo</p>
</div>
</article>
<!-- Unit 2: Different floor plan, no specials, with data-src image -->
<article
data-spaces-id="1002"
data-spaces-unit="CCT-205"
data-spaces-unit-id="5002"
data-spaces-unit-floor="2"
data-spaces-sort-area="950"
data-spaces-sort-bed="2"
data-spaces-sort-bath="1.5"
data-spaces-sort-price="1850"
data-spaces-available="true"
data-spaces-unavailable="false"
data-spaces-soonest="2026-02-15"
data-spaces-sort-date="20260215"
data-spaces-plan-id="202"
data-spaces-sort-plan-name="Birchwood"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100"
data-spaces-href="/units/CCT-205"
data-spaces-inventory-href="/inventory/CCT-205"
data-spaces-specials-content=""
>
<div class="unit-card">
<img data-src="https://example.com/images/unit-205.jpg" alt="Unit 205" />
<h3>Unit CCT-205</h3>
</div>
</article>
<!-- Unit 3: Studio with some missing attributes -->
<article
data-spaces-id="1003"
data-spaces-unit="CCT-310"
data-spaces-unit-id="5003"
data-spaces-unit-floor="3"
data-spaces-sort-area="500"
data-spaces-sort-bed="0"
data-spaces-sort-bath="1"
data-spaces-sort-price="1100"
data-spaces-available="true"
data-spaces-unavailable="false"
data-spaces-soonest="Available Now"
data-spaces-sort-date="20260201"
data-spaces-plan-id="203"
data-spaces-sort-plan-name="Cedar"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100"
data-spaces-href="/units/CCT-310"
data-spaces-inventory-href="/inventory/CCT-310"
>
<div class="unit-card">
<h3>Unit CCT-310</h3>
<p>Studio - $1,100/mo</p>
</div>
</article>
<!-- Unit 4: Unavailable unit -->
<article
data-spaces-id="1004"
data-spaces-unit="CCT-415"
data-spaces-unit-id="5004"
data-spaces-unit-floor="4"
data-spaces-sort-area="1200"
data-spaces-sort-bed="3"
data-spaces-sort-bath="2"
data-spaces-sort-price="2500"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date=""
data-spaces-plan-id="204"
data-spaces-sort-plan-name="Dogwood"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100"
data-spaces-href="/units/CCT-415"
data-spaces-inventory-href="/inventory/CCT-415"
data-spaces-specials-content=""
>
<div class="unit-card">
<img src="https://example.com/images/unit-415.jpg" alt="Unit 415" />
<h3>Unit CCT-415</h3>
</div>
</article>
<!-- Unit 5: Minimal attributes - edge case -->
<article
data-spaces-id="1005"
data-spaces-unit="CCT-520"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-unavailable="false"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100"
>
<div class="unit-card">
<h3>Unit CCT-520</h3>
</div>
</article>
</section>
</div>
</body>
</html>
</html>

2
__tests__/scraper/insertPrices.test.js
View File

@ -22,7 +22,7 @@ let mongoServer;
let client;
let db;
const PRICES_COLLECTION = 'unit_prices_scraper';
const PRICES_COLLECTION = 'unit_prices_migration_test';
// Mock logger for capturing log calls
function createMockLogger() {

2
__tests__/scraper/markStaleUnits.test.js
View File

@ -21,7 +21,7 @@ let mongoServer;
let client;
let db;
const UNITS_COLLECTION = 'units_scraper';
const UNITS_COLLECTION = 'units_migration_test';
// Mock logger for capturing log calls
function createMockLogger() {

289
__tests__/scraper/parseUnits.test.js
View File

@ -87,55 +87,55 @@ describe('parseUnits', () => {
describe('attribute extraction', () => {
it('should extract all expected attributes from a fully populated article', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'W2707');
const unit1 = units.find(u => u.unit_code === 'CCT-101');
expect(unit1).toBeDefined();
// Core identifiers
expect(unit1.id).toBe('154842');
expect(unit1.unit_code).toBe('W2707');
expect(unit1.unit_id).toBe('154842');
expect(unit1.id).toBe('1001');
expect(unit1.unit_code).toBe('CCT-101');
expect(unit1.unit_id).toBe('5001');
// Physical attributes
expect(unit1.floor).toBe('2755');
expect(unit1.area).toBe('1210');
expect(unit1.bed_count).toBe('2');
expect(unit1.bath_count).toBe('2');
expect(unit1.floor).toBe('1');
expect(unit1.area).toBe('750');
expect(unit1.bed_count).toBe('1');
expect(unit1.bath_count).toBe('1');
// Pricing
expect(unit1.price).toBe('5230');
expect(unit1.price).toBe('1450');
// Availability
expect(unit1.available).toBe('true');
expect(unit1.unavailable).toBe('false');
expect(unit1.soonest).toBe('2025-10-11');
expect(unit1.date_available).toBe('1760140800');
expect(unit1.soonest).toBe('2026-03-01');
expect(unit1.date_available).toBe('20260301');
// Plan information
expect(unit1.plan_id).toBe('10270');
expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
expect(unit1.plan_id).toBe('201');
expect(unit1.plan_name).toBe('Alpine');
// Property information
expect(unit1.obj_type).toBe('unit');
expect(unit1.community).toBe('Country Club Towers');
expect(unit1.asset).toBe('420');
expect(unit1.asset).toBe('100');
// URLs
expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
expect(unit1.href).toBe('/units/CCT-101');
expect(unit1.inventory_href).toBe('/inventory/CCT-101');
// Specials
expect(unit1.specials_content).toBe('');
expect(unit1.specials_content).toBe('First month free!');
});
it('should extract correct count of units from the fixture', () => {
const units = parseUnits(sampleHtml, mockLogger);
expect(units).toHaveLength(10);
expect(units).toHaveLength(5);
});
it('should return raw string values without type conversion', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'W2707');
const unit = units.find(u => u.unit_code === 'CCT-101');
// All values should be strings (raw extraction, no conversion)
expect(typeof unit.id).toBe('string');
@ -152,48 +152,24 @@ describe('parseUnits', () => {
// ---------------------------------------------------------------
describe('image URL extraction', () => {
it('should extract image_url from img src attribute', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
<img src="https://example.com/images/unit-1.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'CCT-101');
expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
});
it('should fall back to data-src when src is not present', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
<img data-src="https://example.com/images/unit-2.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const units = parseUnits(sampleHtml, mockLogger);
const unit2 = units.find(u => u.unit_code === 'CCT-205');
expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
});
it('should not set image_url when no img element exists', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
<div>No image here</div>
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const units = parseUnits(sampleHtml, mockLogger);
const unit3 = units.find(u => u.unit_code === 'CCT-310');
expect(units[0].image_url).toBeUndefined();
expect(unit3.image_url).toBeUndefined();
});
});
@ -202,67 +178,37 @@ describe('parseUnits', () => {
// ---------------------------------------------------------------
describe('missing attributes', () => {
it('should return undefined for attributes not present on the article', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
const units = parseUnits(sampleHtml, mockLogger);
// Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
// soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
const unit5 = units.find(u => u.unit_code === 'CCT-520');
expect(unit).toBeDefined();
expect(unit.floor).toBeUndefined();
expect(unit.area).toBeUndefined();
expect(unit.bed_count).toBeUndefined();
expect(unit.bath_count).toBeUndefined();
expect(unit.soonest).toBeUndefined();
expect(unit.date_available).toBeUndefined();
expect(unit.plan_id).toBeUndefined();
expect(unit.plan_name).toBeUndefined();
expect(unit.href).toBeUndefined();
expect(unit.inventory_href).toBeUndefined();
expect(unit.specials_content).toBeUndefined();
expect(unit5).toBeDefined();
expect(unit5.floor).toBeUndefined();
expect(unit5.area).toBeUndefined();
expect(unit5.bed_count).toBeUndefined();
expect(unit5.bath_count).toBeUndefined();
expect(unit5.soonest).toBeUndefined();
expect(unit5.date_available).toBeUndefined();
expect(unit5.plan_id).toBeUndefined();
expect(unit5.plan_name).toBeUndefined();
expect(unit5.href).toBeUndefined();
expect(unit5.inventory_href).toBeUndefined();
expect(unit5.specials_content).toBeUndefined();
});
it('should still extract the attributes that are present on a sparse article', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
const units = parseUnits(sampleHtml, mockLogger);
const unit5 = units.find(u => u.unit_code === 'CCT-520');
expect(unit.id).toBe('1005');
expect(unit.unit_code).toBe('SPARSE-1');
expect(unit.unit_id).toBe('5005');
expect(unit.price).toBe('1300');
expect(unit.available).toBe('true');
expect(unit.obj_type).toBe('unit');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('100');
expect(unit5.id).toBe('1005');
expect(unit5.unit_code).toBe('CCT-520');
expect(unit5.unit_id).toBe('5005');
expect(unit5.price).toBe('1300');
expect(unit5.available).toBe('true');
expect(unit5.obj_type).toBe('unit');
expect(unit5.community).toBe('Country Club Towers');
expect(unit5.asset).toBe('100');
});
});
@ -367,7 +313,7 @@ describe('parseUnits', () => {
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 10 })
expect.objectContaining({ count: 5 })
);
});
@ -384,134 +330,31 @@ describe('parseUnits', () => {
});
// ---------------------------------------------------------------
// 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
// ---------------------------------------------------------------
describe('fixture: sample-listing-empty.html', () => {
let emptyHtml;
beforeAll(() => {
emptyHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-empty.html'),
'utf-8'
);
});
it('should return empty array when fixture has container but no articles', () => {
const units = parseUnits(emptyHtml, mockLogger);
expect(units).toEqual([]);
// Container exists so no error about missing container
expect(mockLogger.error).not.toHaveBeenCalled();
});
it('should log zero parsed units for empty fixture', () => {
parseUnits(emptyHtml, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 0 })
);
});
});
describe('fixture: sample-listing-call.html', () => {
let callHtml;
beforeAll(() => {
callHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-call.html'),
'utf-8'
);
});
it('should extract both units from the call-for-pricing fixture', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units).toHaveLength(2);
expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
});
it('should preserve "Call for pricing" as raw string in the price field', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].price).toBe('Call for pricing');
expect(units[1].price).toBe('Call for pricing');
});
it('should extract all attributes correctly from call-for-pricing units', () => {
const units = parseUnits(callHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'P-101');
expect(unit.id).toBe('9001');
expect(unit.unit_id).toBe('9001');
expect(unit.floor).toBe('1');
expect(unit.area).toBe('750');
expect(unit.bed_count).toBe('1');
expect(unit.bath_count).toBe('1');
expect(unit.available).toBe('true');
expect(unit.unavailable).toBe('false');
expect(unit.soonest).toBe('Now');
expect(unit.date_available).toBe('1706745600');
expect(unit.plan_id).toBe('500');
expect(unit.plan_name).toBe('Penthouse A');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('420');
expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.specials_content).toBe('');
});
it('should not have image_url when fixture units lack img elements', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].image_url).toBeUndefined();
expect(units[1].image_url).toBeUndefined();
});
});
// ---------------------------------------------------------------
// 8. Multiple units with varied data
// 7. Multiple units with varied data
// ---------------------------------------------------------------
describe('varied unit data', () => {
it('should handle unavailable units correctly', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="4001"
data-spaces-unit="UNAVAIL-1"
data-spaces-sort-price="2000"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date="">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
const units = parseUnits(sampleHtml, mockLogger);
const unit4 = units.find(u => u.unit_code === 'CCT-415');
expect(unit.available).toBe('false');
expect(unit.unavailable).toBe('true');
expect(unit.soonest).toBe('');
expect(unit.date_available).toBe('');
expect(unit4.available).toBe('false');
expect(unit4.unavailable).toBe('true');
expect(unit4.soonest).toBe('');
expect(unit4.date_available).toBe('');
});
it('should handle half bath counts', () => {
const units = parseUnits(sampleHtml, mockLogger);
// E3205 is the penthouse with 2.5 baths
const unit = units.find(u => u.unit_code === 'E3205');
const unit2 = units.find(u => u.unit_code === 'CCT-205');
expect(unit.bath_count).toBe('2.5');
expect(unit2.bath_count).toBe('1.5');
});
it('should handle studio units (bed_count = 0)', () => {
const units = parseUnits(sampleHtml, mockLogger);
// W2603 is the studio with bed_count 0
const unit = units.find(u => u.unit_code === 'W2603');
const unit3 = units.find(u => u.unit_code === 'CCT-310');
expect(unit.bed_count).toBe('0');
expect(unit3.bed_count).toBe('0');
});
});
});

16
__tests__/scraper/scraperConfig.test.js
View File

@ -50,15 +50,15 @@ describe('config/scraper', () => {
});
describe('SCRAPER_TIMEZONE', () => {
test('should default to "America/Denver" when env var not set', () => {
test('should default to "UTC" when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
expect(config.SCRAPER_TIMEZONE).toBe('UTC');
});
test('should use env var override when set', () => {
process.env.SCRAPER_TIMEZONE = 'America/New_York';
process.env.SCRAPER_TIMEZONE = 'America/Denver';
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/New_York');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
});
});
@ -142,14 +142,14 @@ describe('config/scraper', () => {
describe('COLLECTIONS', () => {
describe('default values', () => {
test('UNITS should default to "units_scraper"', () => {
test('UNITS should default to "units_migration_test"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
});
test('PRICES should default to "unit_prices_scraper"', () => {
test('PRICES should default to "unit_prices_migration_test"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
});
test('DAILY_SUMMARIES should default to "daily_summaries"', () => {

408
__tests__/scraper/scraperRoutes.test.js
View File

@ -33,7 +33,6 @@
*/
const request = require('supertest');
const jwt = require('jsonwebtoken');
const { MongoClient, ObjectId } = require('mongodb');
const {
createTestApp,
@ -118,8 +117,8 @@ jest.mock('../../config/scraper', () => ({
RETRY_CONFIG: { maxRetries: 3, baseDelay: 1000, timeout: 30000 },
SHUTDOWN_TIMEOUT: 30000,
COLLECTIONS: {
UNITS: 'units_scraper',
PRICES: 'unit_prices_scraper',
UNITS: 'units_migration_test',
PRICES: 'unit_prices_migration_test',
DAILY_SUMMARIES: 'daily_summaries',
SCRAPER_RUNS: 'scraper_runs'
}
@ -1334,407 +1333,4 @@ describe('Scraper Routes', () => {
});
});
});
// ============================================================
// Authentication & Authorization Middleware Tests (SCRAPE-21)
// Comprehensive tests for requireAuth + requireAdmin on all
// /admin/scraper/* routes
// ============================================================
describe('Authentication & Authorization Middleware', () => {
const scraperRoutes = [
{ method: 'post', path: '/api/admin/scraper/run', expectedAdminStatus: 202 },
{ method: 'get', path: '/api/admin/scraper/status', expectedAdminStatus: 200 },
{ method: 'get', path: '/api/admin/scraper/history', expectedAdminStatus: 200 }
];
// ============================================================
// 401 - Missing JWT token
// ============================================================
describe('missing JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} without auth token`, async () => {
const res = await request(app)[method](path)
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 401 - Invalid JWT token
// ============================================================
describe('invalid JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with invalid token`, async () => {
const res = await request(app)[method](path)
.set('Cookie', ['auth_token=this-is-not-a-valid-jwt-token'])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Expired JWT token
// ============================================================
describe('expired JWT token (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with expired token`, async () => {
const adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
// Create a token that expired 1 hour ago
const secret = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
const expiredToken = jwt.sign(
{ userId: adminUser._id.toString() },
secret,
{ expiresIn: '-1h' }
);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${expiredToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Token signed with wrong secret
// ============================================================
describe('token signed with wrong secret (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} with wrong-secret token`, async () => {
const adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
// Sign token with a different secret
const wrongSecretToken = jwt.sign(
{ userId: adminUser._id.toString() },
'completely-wrong-secret-key',
{ expiresIn: '7d' }
);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${wrongSecretToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Invalid token');
});
});
});
// ============================================================
// 401 - Disabled user (isActive: false)
// ============================================================
describe('disabled user (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} when user is disabled`, async () => {
const disabledAdmin = createTestAdmin({ isActive: false });
await insertTestUser(db, disabledAdmin);
const token = generateTestToken(disabledAdmin._id);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 401 - Token for non-existent user
// ============================================================
describe('non-existent user (401)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 401 for ${method.toUpperCase()} ${path} when user does not exist`, async () => {
const nonExistentId = new ObjectId();
const token = generateTestToken(nonExistentId);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(401);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Authentication required');
});
});
});
// ============================================================
// 403 - Non-admin user
// ============================================================
describe('non-admin user (403)', () => {
scraperRoutes.forEach(({ method, path }) => {
it(`should return 403 for ${method.toUpperCase()} ${path} for non-admin user`, async () => {
const regularUser = createTestUser({ role: 'user' });
await insertTestUser(db, regularUser);
const token = generateTestToken(regularUser._id);
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${token}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(403);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toBe('Admin access required');
});
});
});
// ============================================================
// 200/202 - Valid admin user
// ============================================================
describe('valid admin user (200/202)', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
});
scraperRoutes.forEach(({ method, path, expectedAdminStatus }) => {
it(`should return ${expectedAdminStatus} for ${method.toUpperCase()} ${path} for admin user`, async () => {
const res = await request(app)[method](path)
.set('Cookie', [`auth_token=${adminToken}`])
.send(method === 'post' ? {} : undefined);
expect(res.status).toBe(expectedAdminStatus);
expect(res.body).toHaveProperty('data');
});
});
});
// ============================================================
// Rate Limiting for POST /api/admin/scraper/run (SCRAPE-22)
// ============================================================
describe('rate limiting', () => {
let adminUser;
let adminToken;
beforeEach(async () => {
adminUser = createTestAdmin();
await insertTestUser(db, adminUser);
adminToken = generateTestToken(adminUser._id);
// Reset the rate limiter between tests
const { resetRateLimiter } = require('../../routes/admin');
resetRateLimiter();
});
it('should allow the first 5 requests within the rate limit window', async () => {
for (let i = 0; i < 5; i++) {
// Reset scraper state to idle before each request so mutex does not block
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(202);
// Wait for async scrape to release lock
await new Promise(resolve => setTimeout(resolve, 50));
}
});
it('should return 429 when the 6th request exceeds the rate limit', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request should be rate limited
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.body).toHaveProperty('error');
expect(res.body.error).toMatch(/rate limit/i);
});
it('should include Retry-After header in 429 response', async () => {
// Make 5 successful requests
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// 6th request
scraperJob.__reset();
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
expect(res.headers).toHaveProperty('retry-after');
const retryAfter = parseInt(res.headers['retry-after']);
expect(retryAfter).toBeGreaterThan(0);
// Should be less than or equal to 3600 seconds (1 hour)
expect(retryAfter).toBeLessThanOrEqual(3600);
});
it('should track rate limits per user (different admins have separate limits)', async () => {
// Create a second admin
const admin2 = createTestAdmin();
await insertTestUser(db, admin2);
const admin2Token = generateTestToken(admin2._id);
// Use up first admin's rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify first admin is rate limited
scraperJob.__reset();
const res1 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res1.status).toBe(429);
// Second admin should still be able to make requests
scraperJob.__reset();
const res2 = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${admin2Token}`])
.send({});
expect(res2.status).toBe(202);
});
it('should reset rate limit after window expires', async () => {
// Access the rate limiter internals for testing
const adminModule = require('../../routes/admin');
// Use up the rate limit (5 requests)
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Verify rate limited
scraperJob.__reset();
const blockedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(blockedRes.status).toBe(429);
// Simulate window expiry by resetting the rate limiter
adminModule.resetRateLimiter();
// Should be allowed again after window reset
scraperJob.__reset();
const allowedRes = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(allowedRes.status).toBe(202);
});
it('should check rate limit before mutex (rate limit takes precedence)', async () => {
// Use up rate limit
for (let i = 0; i < 5; i++) {
scraperJob.__reset();
await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
await new Promise(resolve => setTimeout(resolve, 50));
}
// Set scraper as running (would normally get 409)
scraperJob._setState(true, 'existing-job');
// Should get 429 (rate limit), not 409 (mutex conflict)
const res = await request(app)
.post('/api/admin/scraper/run')
.set('Cookie', [`auth_token=${adminToken}`])
.send({});
expect(res.status).toBe(429);
});
});
// ============================================================
// Middleware ordering: requireAuth runs before requireAdmin
// ============================================================
describe('middleware ordering', () => {
it('should return 401 (not 403) when token is missing, even for non-admin scenario', async () => {
// Without any token, requireAuth should reject with 401
// before requireAdmin ever runs
const res = await request(app)
.get('/api/admin/scraper/status');
expect(res.status).toBe(401);
expect(res.body.error).toBe('Authentication required');
});
it('should return 401 (not 403) for disabled admin user', async () => {
// A disabled admin should get 401 from requireAuth
// even though they have the admin role
const disabledAdmin = createTestAdmin({ isActive: false, role: 'admin' });
await insertTestUser(db, disabledAdmin);
const token = generateTestToken(disabledAdmin._id);
const res = await request(app)
.get('/api/admin/scraper/status')
.set('Cookie', [`auth_token=${token}`]);
expect(res.status).toBe(401);
expect(res.body.error).toBe('Authentication required');
});
});
});
});

296
__tests__/scraper/scraperService.test.js
View File

@ -20,10 +20,9 @@ let mongoServer;
let client;
let db;
// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
// We will require runScrape and recordScraperRun after implementation
let runScrape;
let recordScraperRun;
let createScraperIndexes;
// Store original module references so we can mock individual functions
let scraperService;
@ -37,7 +36,7 @@ beforeAll(async () => {
// Dynamically require to pick up implementation
scraperService = require('../../services/scraperService');
({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
({ runScrape, recordScraperRun } = scraperService);
});
afterAll(async () => {
@ -220,11 +219,11 @@ describe('runScrape', () => {
expect(result.staleUnitsCount).toBe(0);
// Verify no units were written to the units collection
const unitsCount = await db.collection('units_scraper').countDocuments();
const unitsCount = await db.collection('units_migration_test').countDocuments();
expect(unitsCount).toBe(0);
// Verify no prices were written
const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
expect(pricesCount).toBe(0);
});
@ -278,7 +277,7 @@ describe('runScrape', () => {
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'units_scraper') {
if (name === 'units_migration_test') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
@ -344,7 +343,7 @@ describe('runScrape', () => {
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'unit_prices_scraper') {
if (name === 'unit_prices_migration_test') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
@ -389,7 +388,7 @@ describe('runScrape', () => {
const yesterdayStr = yesterday.toISOString().split('T')[0];
// Yesterday had units: OLD-A, OLD-B, OLD-C
await db.collection('unit_prices_scraper').insertMany([
await db.collection('unit_prices_migration_test').insertMany([
{ unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
{ unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
{ unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
@ -456,284 +455,3 @@ describe('runScrape', () => {
expect(result.errors).toContain('No units found in HTML');
});
});
// ============================================================
// Test: sanitizeError - Error message sanitization
// ============================================================
describe('sanitizeError', () => {
let sanitizeError;
beforeAll(() => {
({ sanitizeError } = require('../../services/scraperService'));
});
// ----------------------------------------------------------
// File path sanitization
// ----------------------------------------------------------
describe('file path sanitization', () => {
it('should remove Unix absolute file paths from error messages', () => {
const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/home\/user/);
expect(sanitized.message).toContain('ENOENT');
});
it('should remove /var paths from error messages', () => {
const error = new Error('Failed to read /var/app/current/data/secrets.yml');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/var\/app/);
expect(sanitized.message).toContain('Failed to read');
});
it('should remove Windows-style file paths from error messages', () => {
const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/C:\\Users/);
expect(sanitized.message).toContain('Cannot find module');
});
it('should remove /tmp and /usr paths from error messages', () => {
const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/tmp\//);
expect(sanitized.message).not.toMatch(/\/usr\//);
});
});
// ----------------------------------------------------------
// MongoDB connection string sanitization
// ----------------------------------------------------------
describe('connection string sanitization', () => {
it('should redact mongodb:// connection strings', () => {
const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('s3cretP4ss');
expect(sanitized.message).not.toContain('admin:');
expect(sanitized.message).toContain('Connection failed');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact mongodb+srv:// connection strings', () => {
const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('password123');
expect(sanitized.message).not.toContain('user:');
expect(sanitized.message).toContain('Timeout connecting to');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact connection string without credentials', () => {
const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
});
// ----------------------------------------------------------
// Credential / secret sanitization
// ----------------------------------------------------------
describe('credential sanitization', () => {
it('should redact common environment variable patterns', () => {
const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('sk-abc123xyz');
expect(sanitized.message).not.toContain('bearer-9876');
});
it('should redact password patterns', () => {
const error = new Error('Auth failed with password=MyS3cret!');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('MyS3cret!');
});
});
// ----------------------------------------------------------
// Error type preservation
// ----------------------------------------------------------
describe('error type preservation', () => {
it('should preserve the error type/name', () => {
const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TypeError');
});
it('should preserve custom error names', () => {
const error = new Error('Timeout at /var/app/scraper.js:100');
error.name = 'TimeoutError';
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TimeoutError');
});
it('should preserve error name for RangeError', () => {
const error = new RangeError('Maximum call stack size exceeded');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('RangeError');
});
});
// ----------------------------------------------------------
// General description preserved for debugging
// ----------------------------------------------------------
describe('general description preservation', () => {
it('should preserve a useful general description', () => {
const error = new Error('Network timeout after 30000ms');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('Network timeout after 30000ms');
});
it('should preserve error description when no sensitive data present', () => {
const error = new Error('Request failed with status code 500');
const sanitized = sanitizeError(error);
expect(sanitized.message).toBe('Request failed with status code 500');
});
it('should return a useful message even after heavy sanitization', () => {
const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('ECONNREFUSED');
expect(sanitized.message.length).toBeGreaterThan(5);
});
});
// ----------------------------------------------------------
// Stack trace sanitization
// ----------------------------------------------------------
describe('stack trace removal', () => {
it('should remove file paths from stack traces', () => {
const error = new Error('Something failed');
error.stack = 'Error: Something failed\n at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
const sanitized = sanitizeError(error);
expect(sanitized.stack).not.toMatch(/\/home\/user/);
expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
});
it('should handle errors without stack trace', () => {
const error = new Error('No stack');
error.stack = undefined;
const sanitized = sanitizeError(error);
expect(sanitized.stack).toBeUndefined();
});
});
// ----------------------------------------------------------
// Integration: sanitization applied before recordScraperRun()
// ----------------------------------------------------------
describe('sanitization before recordScraperRun()', () => {
it('should store sanitized error message in scraper_runs on failure', async () => {
const html = createSampleHtml(['SANITIZE-A']);
// Create a db proxy that throws an error containing sensitive info
const sensitiveError = new Error(
'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
);
sensitiveError.name = 'MongoServerError';
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'units_scraper') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
return async () => { throw sensitiveError; };
}
const value = target[prop];
if (typeof value === 'function') {
return value.bind(target);
}
return value;
}
});
}
return realCollection;
}
};
const result = await runScrape(faultyDb, {
jobId: 'test-sanitized-error',
htmlContent: html
});
expect(result.status).toBe('failed');
// Verify the error stored in result.errors is sanitized
const errorMsg = result.errors[0];
expect(errorMsg).not.toContain('SuperSecret');
expect(errorMsg).not.toContain('admin:');
expect(errorMsg).not.toContain('/home/deploy/');
expect(errorMsg).toContain('MongoServerError');
// Verify the error stored in scraper_runs is sanitized
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('failed');
const storedError = runRecord.errors[0];
expect(storedError).not.toContain('SuperSecret');
expect(storedError).not.toContain('admin:');
expect(storedError).not.toContain('/home/deploy/');
});
});
});
// ============================================================
// Test: createScraperIndexes
// ============================================================
describe('createScraperIndexes', () => {
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
beforeEach(() => {
mockLogger.info.mockClear();
mockLogger.warn.mockClear();
mockLogger.error.mockClear();
});
it('should create the status_startedAt compound index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
expect(statusIndex).toBeTruthy();
expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
});
it('should create the startedAt_desc index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(startedAtIndex).toBeTruthy();
expect(startedAtIndex.key).toEqual({ startedAt: -1 });
});
it('should log success after creating indexes', async () => {
await createScraperIndexes(db, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
});
it('should be idempotent (no error on second call)', async () => {
await createScraperIndexes(db, mockLogger);
// Calling again should not throw
await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
// Indexes should still exist
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(statusIndex).toBeTruthy();
expect(startedAtIndex).toBeTruthy();
});
it('should handle database errors gracefully', async () => {
// Pass an object that will throw when collection() is called
const faultyDb = {
collection: () => {
throw new Error('Connection lost');
}
};
await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
});
});

177
__tests__/scraper/serverIntegration.test.js
View File

@ -1,177 +0,0 @@
/**
* Tests for scraper integration in server.js startup
*
* Covers:
* - Server initializes scraper scheduler on startup
* - Server does not crash if scheduler init fails
* - Indexes are created before scheduler starts
* - Scheduler not initialized when SCRAPER_ENABLED=false
*
* Strategy: Instead of importing server.js directly (which starts Express and
* connects to MongoDB), we replicate the scraper initialization logic from
* connectToMongoDB() and test it with mocked dependencies. This verifies the
* integration contract without needing a full server boot.
*/
const { MongoClient } = require('mongodb');
describe('Server Scraper Integration', () => {
let connection;
let db;
beforeAll(async () => {
const uri = process.env.MONGO_URI;
connection = await MongoClient.connect(uri);
db = connection.db('apartments_server_integration_test');
});
afterAll(async () => {
if (connection) {
await connection.close();
}
});
describe('scraper initialization during startup', () => {
let mockCreateScraperIndexes;
let mockInitializeScheduler;
let mockCreateLogger;
let mockLogger;
beforeEach(() => {
mockLogger = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
debug: jest.fn()
};
mockCreateLogger = jest.fn(() => mockLogger);
mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
mockInitializeScheduler = jest.fn();
});
/**
* Replicates the scraper initialization logic from server.js connectToMongoDB().
* This is the exact pattern used in production:
*
* try {
* const scraperLogger = createLogger('server-init');
* await createScraperIndexes(db, scraperLogger);
* if (scraperConfig.SCRAPER_ENABLED) {
* initializeScheduler(db);
* }
* } catch (error) {
* console.error('Scraper initialization failed (server continues):', error.message);
* }
*/
async function runScraperInit(testDb, config) {
try {
const scraperLogger = mockCreateLogger('server-init');
await mockCreateScraperIndexes(testDb, scraperLogger);
if (config.SCRAPER_ENABLED) {
mockInitializeScheduler(testDb);
}
return { success: true };
} catch (error) {
return { success: false, error: error.message };
}
}
it('should initialize scraper scheduler on startup when enabled', async () => {
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
});
it('should not crash if createScraperIndexes throws an error', async () => {
mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
// The server should continue (error is caught, not thrown)
expect(result.success).toBe(false);
expect(result.error).toBe('Index creation failed');
// initializeScheduler should NOT have been called since the error occurred before it
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
it('should create indexes before initializing the scheduler', async () => {
const callOrder = [];
mockCreateScraperIndexes.mockImplementation(async () => {
callOrder.push('createScraperIndexes');
});
mockInitializeScheduler.mockImplementation(() => {
callOrder.push('initializeScheduler');
});
const config = { SCRAPER_ENABLED: true };
await runScraperInit(db, config);
expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
});
it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
const config = { SCRAPER_ENABLED: false };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
});
describe('server.js imports and exports verification', () => {
it('should be able to import createScraperIndexes from scraperService', () => {
const { createScraperIndexes } = require('../../services/scraperService');
expect(typeof createScraperIndexes).toBe('function');
});
it('should be able to import initializeScheduler from scraperJob', () => {
const { initializeScheduler } = require('../../jobs/scraperJob');
expect(typeof initializeScheduler).toBe('function');
});
it('should be able to import registerSignalHandlers from scraperJob', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
expect(typeof registerSignalHandlers).toBe('function');
});
it('should be able to import SCRAPER_ENABLED from scraper config', () => {
const scraperConfig = require('../../config/scraper');
expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
});
});
describe('createScraperIndexes error handling with real db', () => {
it('should handle index creation gracefully with a valid db', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const { createLogger } = require('../../services/scraperLogger');
const logger = createLogger('test-server-init');
// Should not throw - creates indexes on the test db
await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
});
it('should throw when given an invalid db object', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
// Passing null should throw (simulates what the try/catch in server.js protects against)
await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
});
});
describe('registerSignalHandlers', () => {
it('should register SIGTERM and SIGINT handlers without throwing', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
// Should not throw when called
expect(() => registerSignalHandlers()).not.toThrow();
});
});
});

2
__tests__/scraper/upsertUnits.test.js
View File

@ -56,7 +56,7 @@ beforeEach(async () => {
});
describe('upsertUnits', () => {
const UNITS_COLLECTION = 'units_scraper';
const UNITS_COLLECTION = 'units_migration_test';
// ---------------------------------------------------------------
// 1. bulkWrite is called with correct updateOne operations

9
config/scraper.js
View File

@ -13,7 +13,7 @@ module.exports = {
// Scheduling configuration
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings
@ -30,11 +30,10 @@ module.exports = {
// Graceful shutdown timeout (how long to wait for running job before force-stopping)
SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
// MongoDB collection names - defaults to validation collections for scraper validation;
// switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
// MongoDB collection names (environment variable overrides for development isolation)
COLLECTIONS: {
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper',
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper',
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
}

81
routes/admin.js
View File

@ -1156,59 +1156,8 @@ const {
getNextScheduledRun
} = require('../jobs/scraperJob');
const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger');
const scraperConfig = require('../config/scraper');
// Logger for scraper admin routes
const scraperRouteLogger = createLogger('scraper-admin');
// Rate limiting for manual scrape trigger (per-user, in-memory)
const RATE_LIMIT_MAX_REQUESTS = 5;
const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
const rateLimitStore = new Map();
/**
* Check rate limit for a given user ID.
* Returns an object indicating whether the request is allowed.
*
* @param {string} userId - The user ID to check
* @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
*/
function checkRateLimit(userId) {
const now = Date.now();
const userKey = userId.toString();
if (!rateLimitStore.has(userKey)) {
rateLimitStore.set(userKey, []);
}
const timestamps = rateLimitStore.get(userKey);
// Remove timestamps outside the current window
const windowStart = now - RATE_LIMIT_WINDOW_MS;
const validTimestamps = timestamps.filter(ts => ts > windowStart);
rateLimitStore.set(userKey, validTimestamps);
if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
// Calculate when the oldest request in the window will expire
const oldestTimestamp = validTimestamps[0];
const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
return { allowed: false, retryAfterSeconds };
}
// Record this request
validTimestamps.push(now);
return { allowed: true, retryAfterSeconds: null };
}
/**
* Reset the rate limiter (for testing)
*/
function resetRateLimiter() {
rateLimitStore.clear();
}
/**
* POST /api/admin/scraper/run
* Trigger a manual scrape
@ -1222,15 +1171,6 @@ router.post('/scraper/run', async (req, res) => {
const db = req.app.locals.db;
const { dryRun = false, htmlContent = null } = req.body || {};
// Check rate limit (per-user, before mutex check)
const rateLimitResult = checkRateLimit(req.user._id);
if (!rateLimitResult.allowed) {
res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
return res.status(429).json({
error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
});
}
// Check if scraper is already running
if (isScraperRunning()) {
return res.status(409).json({ error: 'Scrape already in progress' });
@ -1250,10 +1190,7 @@ router.post('/scraper/run', async (req, res) => {
// Start scrape asynchronously (do not await - return 202 immediately)
runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
.catch((error) => {
scraperRouteLogger.error('Async scrape failed', {
errorType: error.name,
errorMessage: error.message
});
console.error('Async scrape failed:', error.message);
})
.finally(() => releaseLock());
@ -1268,10 +1205,7 @@ router.post('/scraper/run', async (req, res) => {
});
} catch (error) {
scraperRouteLogger.error('Error triggering scrape', {
errorType: error.name,
errorMessage: error.message
});
console.error('Error triggering scrape:', error);
res.status(500).json({ error: 'Failed to start scrape job' });
}
});
@ -1311,10 +1245,7 @@ router.get('/scraper/status', async (req, res) => {
});
} catch (error) {
scraperRouteLogger.error('Error fetching scraper status', {
errorType: error.name,
errorMessage: error.message
});
console.error('Error fetching scraper status:', error);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
@ -1364,14 +1295,10 @@ router.get('/scraper/history', async (req, res) => {
res.json({ data: history });
} catch (error) {
scraperRouteLogger.error('Error fetching scraper history', {
errorType: error.name,
errorMessage: error.message
});
console.error('Error fetching scraper history:', error);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
module.exports = router;
module.exports.clearStatsCache = clearStatsCache;
module.exports.resetRateLimiter = resetRateLimiter;

35
server.js
View File

@ -11,9 +11,6 @@ const activityRoutes = require('./routes/activity');
const adminRoutes = require('./routes/admin');
const { createIndexes } = require('./models/user');
const { createActivityIndexes } = require('./services/activityLogger');
const { createScraperIndexes } = require('./services/scraperService');
const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
const scraperConfig = require('./config/scraper');
const app = express();
const PORT = process.env.PORT || 3000;
@ -83,32 +80,14 @@ async function connectToMongoDB() {
await createIndexes(db);
await createActivityIndexes(db);
console.log('✅ Passport configured and indexes created');
// Initialize scraper (indexes + scheduler) - errors logged but don't crash server
try {
const { createLogger } = require('./services/scraperLogger');
const scraperLogger = createLogger('server-init');
await createScraperIndexes(db, scraperLogger);
if (scraperConfig.SCRAPER_ENABLED) {
initializeScheduler(db);
console.log('✅ Scraper scheduler initialized');
} else {
console.log('ℹ️ Scraper scheduling disabled');
}
} catch (error) {
console.error('⚠️ Scraper initialization failed (server continues):', error.message);
}
} catch (error) {
console.error('❌ Failed to connect to MongoDB:', error);
process.exit(1);
}
}
// Register scraper signal handlers for graceful shutdown
registerSignalHandlers();
// Helper function to get today's date in configured timezone
const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
// Helper function to get today's date in UTC
const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
// Cache for the most recent date with data
let cachedLatestDate = null;
@ -143,7 +122,7 @@ const getLatestDateWithData = async () => {
}
// Fallback to UTC today if no data found
return getTodayDate();
return getTodayDateUTC();
};
// Helper function to get yesterday relative to the latest date with data
@ -641,10 +620,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
]).toArray();
// 4. Historical Comparison - current vs 30/90/365 day averages
const tz = scraperConfig.SCRAPER_TIMEZONE;
const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const now = new Date();
const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const [currentAvg, avg30, avg90, avg365] = await Promise.all([
db.collection(PRICES_COLLECTION).aggregate([

100
services/scraperService.js
View File

@ -626,11 +626,11 @@ async function updateDailySummary(db, summaryData, logger) {
// ============================================================
/**
* Get today's date in YYYY-MM-DD format using the configured timezone.
* Get today's date in YYYY-MM-DD format (UTC).
* @returns {string} Today's date string
*/
function getToday() {
return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
function getTodayUTC() {
return new Date().toISOString().split('T')[0];
}
/**
@ -651,54 +651,6 @@ async function getYesterdayUnitCodes(db, today) {
return new Set(yesterdayRecords.map(r => r.unit_code));
}
// ============================================================
// Error Sanitization
// ============================================================
/**
* Sanitize an error object to remove sensitive information before storage.
* Removes file paths, connection strings, and credential patterns while
* preserving the error type and a useful general description for debugging.
*
* @param {Error} error - Error object to sanitize
* @returns {Object} Sanitized error with name, message, and optionally stack
*/
function sanitizeError(error) {
const sanitized = {
name: error.name || 'Error',
message: sanitizeMessage(error.message || ''),
};
if (error.stack) {
sanitized.stack = sanitizeMessage(error.stack);
}
return sanitized;
}
/**
* Sanitize a string message by removing sensitive patterns.
* @param {string} message - Raw error message
* @returns {string} Sanitized message
*/
function sanitizeMessage(message) {
let result = message;
// Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
// Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
// Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
// Remove Windows-style absolute paths (C:\..., D:\...)
result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
return result;
}
// ============================================================
// Scraper Run History
// ============================================================
@ -730,37 +682,6 @@ async function recordScraperRun(db, runData, logger) {
}
}
// ============================================================
// Index Management
// ============================================================
/**
* Create indexes on the scraper_runs collection.
* Should be called once during application startup.
* Idempotent - safe to call multiple times.
*
* @param {Db} db - MongoDB database instance
* @param {Object} logger - Logger instance
* @returns {Promise<void>}
*/
async function createScraperIndexes(db, logger) {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
// Compound index for querying runs by status sorted by most recent
await collection.createIndex(
{ status: 1, startedAt: -1 },
{ name: 'status_startedAt' }
);
// Index for sorting all runs by start time (most recent first)
await collection.createIndex(
{ startedAt: -1 },
{ name: 'startedAt_desc' }
);
logger.info('Scraper indexes created successfully');
}
// ============================================================
// Main Orchestration Function
// ============================================================
@ -820,7 +741,7 @@ async function runScrape(db, options = {}) {
result.unitsProcessed = units.length;
// Step 4: Database operations
const today = getToday();
const today = getTodayUTC();
// Get yesterday's unit codes for comparison
const yesterdayUnits = await getYesterdayUnitCodes(db, today);
@ -866,13 +787,12 @@ async function runScrape(db, options = {}) {
result.status = 'success';
} catch (error) {
const cleanError = sanitizeError(error);
logger.error('Scrape failed', {
errorType: cleanError.name,
errorMessage: cleanError.message
errorType: error.name,
errorMessage: error.message
});
result.status = 'failed';
result.errors.push(cleanError.message);
result.errors.push(error.message);
} finally {
result.completedAt = new Date().toISOString();
@ -902,9 +822,8 @@ module.exports = {
markStaleUnits,
updateDailySummary,
recordScraperRun,
createScraperIndexes,
// Export helpers for testing
getToday,
getTodayUTC,
getYesterdayUnitCodes,
getYesterday,
parseInteger,
@ -915,6 +834,5 @@ module.exports = {
parseFloatValue,
trimString,
isRetryableError,
sleep,
sanitizeError
sleep
};