stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

Compare commits

base: stephen:dev
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field
..
compare: stephen:d14bd724ed6d1b8c658460db5e9b0f522be36711
Branches Tags
stephen:main stephen:dev stephen:add-soonest-field

2 Commits
dev ... d14bd724ed

Author SHA1 Message Date
Stephen Minakian
d14bd724ed Add scraperLogger.js with credential redaction
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 14s
Details
CI/CD Pipeline - Apartment API / Run Tests (pull_request) Successful in 9m45s
Details
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
Details
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
Details 
Implement structured JSON logging module for the scraper service:
- createLogger(jobId) factory function for job-scoped logging
- info(), warn(), error() methods with consistent JSON output
- Automatic redaction of MongoDB connection string credentials
- Redaction of sensitive keys (password, secret, token, apikey, authorization)
- Message truncation to prevent log bloat (1000 char limit)
- Circular reference handling
- Buffer object representation as [Buffer: N bytes]

Includes comprehensive test suite with 35 test cases covering:
- Log entry format validation (timestamp, level, message, jobId, context)
- MongoDB URI redaction for both mongodb:// and mongodb+srv://
- Sensitive key redaction
- Edge cases (null, undefined, nested objects, arrays)
- Buffer and circular reference handling
 2026-01-28 00:29:14 -07:00
Stephen Minakian
59fb6da5fb T-33: Add npm dependencies 2026-01-27 23:38:34 -07:00
25 changed files with 30 additions and 11241 deletions
Expand all files Collapse all files

163
.github/workflows/deploy.yml vendored
View File

@ -4,6 +4,7 @@ on:
push: push:
branches: [ main ] branches: [ main ]
pull_request: pull_request:
branches: [ main ]
workflow_dispatch: workflow_dispatch:
env: env:
@ -12,17 +13,22 @@ env:
jobs: jobs:
# ============================================================ # ============================================================
# CI Job - Lint + Test in a single job (one checkout, one npm ci) # Test Job - Runs first, blocks everything if tests fail
# ============================================================ # ============================================================
ci: test:
name: Lint & Test name: Run Tests
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: services:
lint_status: ${{ steps.lint.outcome }} mongodb:
lint_output: ${{ steps.lint.outputs.lint_output }} image: mongo:7
test_status: ${{ steps.test.outcome }} ports:
test_output: ${{ steps.test.outputs.test_output }} - 27018:27017
options: >-
--health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps: steps:
- name: Checkout code - name: Checkout code
@ -37,137 +43,13 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: npm ci run: npm ci
- name: Run ESLint
id: lint
continue-on-error: true
run: |
set +e
OUTPUT=$(npm run lint 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "lint_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
- name: Run tests - name: Run tests
id: test run: npm test -- --runInBand
continue-on-error: true
run: |
set +e
OUTPUT=$(npm test -- --runInBand 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "test_output<<EOF" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
env: env:
MONGO_URI: mongodb://localhost:27018
JWT_SECRET: test-jwt-secret-for-ci JWT_SECRET: test-jwt-secret-for-ci
NODE_ENV: test NODE_ENV: test
# ============================================================
# Notify Job - Send results to n8n webhook
# ============================================================
notify:
name: Send Webhook Notification
runs-on: ubuntu-latest
needs: [ci]
if: always()
steps:
- name: Send results to n8n webhook
env:
LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
TEST_STATUS: ${{ needs.ci.outputs.test_status }}
RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
GH_REPO: ${{ github.repository }}
GH_BRANCH: ${{ github.head_ref || github.ref_name }}
GH_SHA: ${{ github.sha }}
GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
GH_ACTOR: ${{ github.actor }}
GH_EVENT: ${{ github.event_name }}
GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
GH_RUN_ID: ${{ github.run_id }}
GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
run: |
# Determine overall status
if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
OVERALL_STATUS="success"
else
OVERALL_STATUS="failure"
fi
# Outputs are already truncated at the source (ci job)
LINT_OUTPUT="$RAW_LINT_OUTPUT"
TEST_OUTPUT="$RAW_TEST_OUTPUT"
# Build JSON payload
PAYLOAD=$(jq -n \
--arg repo "$GH_REPO" \
--arg branch "$GH_BRANCH" \
--arg commit "$GH_SHA" \
--arg commit_short "${GH_SHA:0:7}" \
--arg commit_message "$GH_COMMIT_MSG" \
--arg author "$GH_ACTOR" \
--arg event "$GH_EVENT" \
--arg pr_number "$GH_PR_NUMBER" \
--arg run_id "$GH_RUN_ID" \
--arg run_url "$GH_RUN_URL" \
--arg overall_status "$OVERALL_STATUS" \
--arg lint_status "$LINT_STATUS" \
--arg lint_output "$LINT_OUTPUT" \
--arg test_status "$TEST_STATUS" \
--arg test_output "$TEST_OUTPUT" \
--arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
'{
repository: $repo,
branch: $branch,
pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
commit: {
sha: $commit,
short_sha: $commit_short,
message: $commit_message,
author: $author
},
event: $event,
run: {
id: $run_id,
url: $run_url
},
status: $overall_status,
results: {
lint: {
status: $lint_status,
output: (if $lint_status != "success" then $lint_output else null end)
},
test: {
status: $test_status,
output: (if $test_status != "success" then $test_output else null end)
}
},
timestamp: $timestamp
}')
# Send webhook
curl -X POST \
-H "Content-Type: application/json" \
-d "$PAYLOAD" \
"$WEBHOOK_URL" \
--fail --silent --show-error
- name: Fail if lint or tests failed
if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
run: |
echo "❌ Pipeline failed:"
echo " Lint: ${{ needs.ci.outputs.lint_status }}"
echo " Test: ${{ needs.ci.outputs.test_status }}"
exit 1
# ============================================================ # ============================================================
# Dependency Scan Job - Runs in parallel with tests # Dependency Scan Job - Runs in parallel with tests
# ============================================================ # ============================================================
@ -198,7 +80,7 @@ jobs:
build: build:
name: Build & Push Image name: Build & Push Image
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [ci, scan-deps, notify] needs: [test, scan-deps]
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
outputs: outputs:
@ -264,13 +146,12 @@ jobs:
# ============================================================ # ============================================================
# SBOM Generation with Syft # SBOM Generation with Syft
# ============================================================ # ============================================================
- name: Install Syft
run: |
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
- name: Generate SBOM with Syft - name: Generate SBOM with Syft
run: | uses: anchore/sbom-action@v0
syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json with:
image: ${{ steps.set-tag.outputs.full_image }}
format: spdx-json
output-file: sbom.spdx.json
# ============================================================ # ============================================================
# Image Signing with Cosign # Image Signing with Cosign

496
__tests__/scraper/dataConversion.test.js
View File

@ -1,496 +0,0 @@
/**
* Tests for convertDataTypes() and its internal helper functions
*
* Covers:
* - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
* - parseInteger: valid integers, null/empty/invalid
* - parsePositiveInteger: treats 0 as null (for area)
* - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
* - parseFloat: "1.5" for half baths, null/empty/invalid
* - parseBoolean: "true"/"false"/"1"/"0", null/empty
* - trimString: trim whitespace, null for empty, null/undefined
* - convertDataTypes: full integration applying correct parser to each field
*/
const {
convertDataTypes,
parseInteger,
parsePositiveInteger,
parseIntegerOrString,
parsePrice,
parseBoolean,
parseFloatValue,
trimString
} = require('../../services/scraperService');
// ---------------------------------------------------------------
// 1. parsePrice
// ---------------------------------------------------------------
describe('parsePrice', () => {
it('should parse "$1,234" format to 1234', () => {
expect(parsePrice('$1,234')).toBe(1234);
});
it('should parse plain numeric string', () => {
expect(parsePrice('1450')).toBe(1450);
});
it('should parse "$2,500.00" stripping non-numeric chars', () => {
expect(parsePrice('$2,500.00')).toBe(2500);
});
it('should return null for "Call for pricing"', () => {
expect(parsePrice('Call for pricing')).toBeNull();
});
it('should return null for "Contact us"', () => {
expect(parsePrice('Contact us')).toBeNull();
});
it('should return null for "Please inquire"', () => {
expect(parsePrice('Please inquire')).toBeNull();
});
it('should return null for negative values', () => {
// After stripping non-numeric chars, "-1500" becomes "1500"
// but if someone passes a numeric -1500 directly, check that too
expect(parsePrice('-')).toBeNull();
});
it('should return null for null', () => {
expect(parsePrice(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parsePrice('')).toBeNull();
});
it('should return null for undefined', () => {
expect(parsePrice(undefined)).toBeNull();
});
it('should return null for completely non-numeric string', () => {
expect(parsePrice('abc')).toBeNull();
});
it('should handle price of 0', () => {
expect(parsePrice('0')).toBe(0);
});
it('should handle large prices', () => {
expect(parsePrice('$15,500')).toBe(15500);
});
});
// ---------------------------------------------------------------
// 2. parseInteger
// ---------------------------------------------------------------
describe('parseInteger', () => {
it('should parse valid integer string', () => {
expect(parseInteger('42')).toBe(42);
});
it('should parse "0" to 0', () => {
expect(parseInteger('0')).toBe(0);
});
it('should parse negative integer string', () => {
expect(parseInteger('-5')).toBe(-5);
});
it('should return null for null', () => {
expect(parseInteger(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parseInteger('')).toBeNull();
});
it('should return null for undefined', () => {
expect(parseInteger(undefined)).toBeNull();
});
it('should return null for non-numeric string', () => {
expect(parseInteger('abc')).toBeNull();
});
it('should return null for NaN', () => {
expect(parseInteger('NaN')).toBeNull();
});
it('should return null for Infinity', () => {
expect(parseInteger('Infinity')).toBeNull();
});
it('should truncate float strings to integer', () => {
expect(parseInteger('3.7')).toBe(3);
});
});
// ---------------------------------------------------------------
// 3. parsePositiveInteger
// ---------------------------------------------------------------
describe('parsePositiveInteger', () => {
it('should parse positive integer normally', () => {
expect(parsePositiveInteger('750')).toBe(750);
});
it('should treat 0 as null (for area field)', () => {
expect(parsePositiveInteger('0')).toBeNull();
});
it('should return null for null input', () => {
expect(parsePositiveInteger(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parsePositiveInteger('')).toBeNull();
});
it('should return null for non-numeric string', () => {
expect(parsePositiveInteger('abc')).toBeNull();
});
it('should handle negative values (parsed as negative int, not positive)', () => {
expect(parsePositiveInteger('-5')).toBe(-5);
});
});
// ---------------------------------------------------------------
// 4. parseIntegerOrString
// ---------------------------------------------------------------
describe('parseIntegerOrString', () => {
it('should return integer when value is cleanly parseable', () => {
expect(parseIntegerOrString('5001')).toBe(5001);
});
it('should return trimmed string for non-numeric value', () => {
expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
});
it('should return trimmed string when value has non-numeric suffix', () => {
expect(parseIntegerOrString('123abc')).toBe('123abc');
});
it('should return null for null', () => {
expect(parseIntegerOrString(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parseIntegerOrString('')).toBeNull();
});
it('should return null for undefined', () => {
expect(parseIntegerOrString(undefined)).toBeNull();
});
it('should trim whitespace before checking', () => {
expect(parseIntegerOrString(' 5001 ')).toBe(5001);
});
it('should return trimmed string for non-numeric with whitespace', () => {
expect(parseIntegerOrString(' abc ')).toBe('abc');
});
});
// ---------------------------------------------------------------
// 5. parseFloatValue
// ---------------------------------------------------------------
describe('parseFloatValue', () => {
it('should parse "1.5" for half baths', () => {
expect(parseFloatValue('1.5')).toBe(1.5);
});
it('should parse "2.0" to 2', () => {
expect(parseFloatValue('2.0')).toBe(2.0);
});
it('should parse integer string as float', () => {
expect(parseFloatValue('3')).toBe(3);
});
it('should return null for null', () => {
expect(parseFloatValue(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parseFloatValue('')).toBeNull();
});
it('should return null for undefined', () => {
expect(parseFloatValue(undefined)).toBeNull();
});
it('should return null for non-numeric string', () => {
expect(parseFloatValue('abc')).toBeNull();
});
it('should return null for NaN', () => {
expect(parseFloatValue('NaN')).toBeNull();
});
it('should return null for Infinity', () => {
expect(parseFloatValue('Infinity')).toBeNull();
});
});
// ---------------------------------------------------------------
// 6. parseBoolean
// ---------------------------------------------------------------
describe('parseBoolean', () => {
it('should parse "true" to true', () => {
expect(parseBoolean('true')).toBe(true);
});
it('should parse "false" to false', () => {
expect(parseBoolean('false')).toBe(false);
});
it('should parse "1" to true', () => {
expect(parseBoolean('1')).toBe(true);
});
it('should parse "0" to false', () => {
expect(parseBoolean('0')).toBe(false);
});
it('should parse "TRUE" (case insensitive) to true', () => {
expect(parseBoolean('TRUE')).toBe(true);
});
it('should parse "False" (case insensitive) to false', () => {
expect(parseBoolean('False')).toBe(false);
});
it('should pass through boolean true', () => {
expect(parseBoolean(true)).toBe(true);
});
it('should pass through boolean false', () => {
expect(parseBoolean(false)).toBe(false);
});
it('should return null for null', () => {
expect(parseBoolean(null)).toBeNull();
});
it('should return null for empty string', () => {
expect(parseBoolean('')).toBeNull();
});
it('should return null for undefined', () => {
expect(parseBoolean(undefined)).toBeNull();
});
it('should return null for unrecognized string', () => {
expect(parseBoolean('yes')).toBeNull();
});
});
// ---------------------------------------------------------------
// 7. trimString
// ---------------------------------------------------------------
describe('trimString', () => {
it('should trim leading and trailing whitespace', () => {
expect(trimString(' hello ')).toBe('hello');
});
it('should return the string as-is if no whitespace', () => {
expect(trimString('hello')).toBe('hello');
});
it('should return null for empty string', () => {
expect(trimString('')).toBeNull();
});
it('should return null for whitespace-only string', () => {
expect(trimString(' ')).toBeNull();
});
it('should return null for null', () => {
expect(trimString(null)).toBeNull();
});
it('should return null for undefined', () => {
expect(trimString(undefined)).toBeNull();
});
it('should convert non-string to string then trim', () => {
expect(trimString(123)).toBe('123');
});
});
// ---------------------------------------------------------------
// 8. convertDataTypes - Full integration test
// ---------------------------------------------------------------
describe('convertDataTypes', () => {
it('should apply correct parser to each field', () => {
const rawUnit = {
id: '1001',
unit_code: 'CCT-101',
unit_id: '5001',
floor: '1',
area: '750',
bed_count: '1',
bath_count: '1.5',
price: '1450',
available: 'true',
unavailable: 'false',
soonest: '2026-03-01',
date_available: '20260301',
plan_id: '201',
plan_name: 'Alpine',
obj_type: 'unit',
community: 'Country Club Towers',
asset: '100',
href: '/units/CCT-101',
inventory_href: '/inventory/CCT-101',
image_url: 'https://example.com/images/unit-101.jpg',
specials_content: 'First month free!'
};
const converted = convertDataTypes(rawUnit);
// Integer fields
expect(converted.id).toBe(1001);
expect(converted.floor).toBe(1);
expect(converted.bed_count).toBe(1);
expect(converted.plan_id).toBe(201);
expect(converted.asset).toBe(100);
expect(converted.date_available).toBe(20260301);
// parseIntegerOrString
expect(converted.unit_id).toBe(5001);
// parsePositiveInteger (area)
expect(converted.area).toBe(750);
// Float field (bath_count)
expect(converted.bath_count).toBe(1.5);
// Price field
expect(converted.price).toBe(1450);
// Boolean fields
expect(converted.available).toBe(true);
expect(converted.unavailable).toBe(false);
// String fields (trimmed)
expect(converted.unit_code).toBe('CCT-101');
expect(converted.plan_name).toBe('Alpine');
expect(converted.soonest).toBe('2026-03-01');
expect(converted.obj_type).toBe('unit');
expect(converted.community).toBe('Country Club Towers');
expect(converted.href).toBe('/units/CCT-101');
expect(converted.inventory_href).toBe('/inventory/CCT-101');
expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
expect(converted.specials_content).toBe('First month free!');
});
it('should preserve string fields without converting them', () => {
const rawUnit = {
unit_code: 'CCT-205',
plan_name: ' Birch ',
soonest: 'Now',
obj_type: 'unit',
community: ' Country Club Towers ',
href: '/units/CCT-205',
inventory_href: '/inventory/CCT-205',
image_url: 'https://example.com/img.jpg',
specials_content: null
};
const converted = convertDataTypes(rawUnit);
expect(converted.unit_code).toBe('CCT-205');
expect(converted.plan_name).toBe('Birch');
expect(converted.community).toBe('Country Club Towers');
expect(converted.specials_content).toBeNull();
});
it('should handle a unit with all null/undefined/empty values', () => {
const rawUnit = {
id: null,
unit_code: undefined,
unit_id: '',
floor: null,
area: '',
bed_count: undefined,
bath_count: null,
price: '',
available: null,
unavailable: undefined,
soonest: '',
date_available: null,
plan_id: undefined,
plan_name: '',
obj_type: null,
community: undefined,
asset: '',
href: null,
inventory_href: undefined,
image_url: '',
specials_content: null
};
const converted = convertDataTypes(rawUnit);
expect(converted.id).toBeNull();
expect(converted.unit_code).toBeNull();
expect(converted.unit_id).toBeNull();
expect(converted.floor).toBeNull();
expect(converted.area).toBeNull();
expect(converted.bed_count).toBeNull();
expect(converted.bath_count).toBeNull();
expect(converted.price).toBeNull();
expect(converted.available).toBeNull();
expect(converted.unavailable).toBeNull();
expect(converted.soonest).toBeNull();
expect(converted.date_available).toBeNull();
expect(converted.plan_id).toBeNull();
expect(converted.plan_name).toBeNull();
expect(converted.obj_type).toBeNull();
expect(converted.community).toBeNull();
expect(converted.asset).toBeNull();
expect(converted.href).toBeNull();
expect(converted.inventory_href).toBeNull();
expect(converted.image_url).toBeNull();
expect(converted.specials_content).toBeNull();
});
it('should handle area=0 as null (parsePositiveInteger)', () => {
const rawUnit = { area: '0' };
const converted = convertDataTypes(rawUnit);
expect(converted.area).toBeNull();
});
it('should handle bed_count=0 as valid (studio)', () => {
const rawUnit = { bed_count: '0' };
const converted = convertDataTypes(rawUnit);
expect(converted.bed_count).toBe(0);
});
it('should handle "Call for pricing" in price field', () => {
const rawUnit = { price: 'Call for pricing' };
const converted = convertDataTypes(rawUnit);
expect(converted.price).toBeNull();
});
it('should handle "$1,234" price format', () => {
const rawUnit = { price: '$1,234' };
const converted = convertDataTypes(rawUnit);
expect(converted.price).toBe(1234);
});
it('should handle unit_id as string identifier', () => {
const rawUnit = { unit_id: 'UNIT-ABC' };
const converted = convertDataTypes(rawUnit);
expect(converted.unit_id).toBe('UNIT-ABC');
});
it('should handle bath_count with half baths', () => {
const rawUnit = { bath_count: '1.5' };
const converted = convertDataTypes(rawUnit);
expect(converted.bath_count).toBe(1.5);
});
});

535
__tests__/scraper/fetchPage.test.js
View File

@ -1,535 +0,0 @@
/**
* Tests for fetchPage function
*
* Acceptance Criteria:
* - Uses axios for HTTP GET
* - Enforces timeout from config (default 30s)
* - Sets User-Agent header from config
* - Implements retry with exponential backoff (1s, 2s, 4s)
* - Retries on 5xx errors and network timeouts
* - Does NOT retry on 4xx errors
* - Logs each attempt with attempt number
* - Returns HTML string on success
* - Throws error after all retries exhausted
*/
const axios = require('axios');
// Mock axios
jest.mock('axios');
// Mock config
jest.mock('../../config/scraper', () => ({
RETRY_CONFIG: {
maxRetries: 3,
baseDelay: 1000,
timeout: 30000
},
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
}));
// Import after mocking
const { fetchPage } = require('../../services/scraperService');
const config = require('../../config/scraper');
describe('fetchPage', () => {
let mockLogger;
beforeEach(() => {
jest.clearAllMocks();
jest.useFakeTimers();
// Create mock logger
mockLogger = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
});
afterEach(() => {
jest.useRealTimers();
});
describe('successful requests', () => {
it('should return HTML string on success', async () => {
const htmlContent = '<html><body>Test content</body></html>';
axios.get.mockResolvedValueOnce({
status: 200,
data: htmlContent
});
const result = await fetchPage('https://example.com', mockLogger);
expect(result).toBe(htmlContent);
expect(axios.get).toHaveBeenCalledTimes(1);
});
it('should use axios for HTTP GET', async () => {
const url = 'https://example.com/apartments';
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage(url, mockLogger);
expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
});
it('should set timeout from config', async () => {
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage('https://example.com', mockLogger);
expect(axios.get).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
timeout: config.RETRY_CONFIG.timeout
})
);
});
it('should set User-Agent header from config', async () => {
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage('https://example.com', mockLogger);
expect(axios.get).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
headers: expect.objectContaining({
'User-Agent': config.USER_AGENT
})
})
);
});
it('should log the attempt with attempt number', async () => {
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage('https://example.com', mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Fetching page',
expect.objectContaining({
url: 'https://example.com',
attempt: 1
})
);
});
it('should log successful fetch with status and content length', async () => {
const htmlContent = '<html><body>Content</body></html>';
axios.get.mockResolvedValueOnce({
status: 200,
data: htmlContent
});
await fetchPage('https://example.com', mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Page fetched successfully',
expect.objectContaining({
status: 200,
contentLength: htmlContent.length
})
);
});
});
describe('retry behavior on 5xx errors', () => {
it('should retry on 500 Internal Server Error', async () => {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
axios.get
.mockRejectedValueOnce(error500)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
// Fast-forward through the retry delay
await jest.advanceTimersByTimeAsync(1000);
const result = await promise;
expect(result).toBe('<html></html>');
expect(axios.get).toHaveBeenCalledTimes(2);
});
it('should retry on 502 Bad Gateway', async () => {
const error502 = new Error('Bad Gateway');
error502.response = { status: 502 };
axios.get
.mockRejectedValueOnce(error502)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(2);
});
it('should retry on 503 Service Unavailable', async () => {
const error503 = new Error('Service Unavailable');
error503.response = { status: 503 };
axios.get
.mockRejectedValueOnce(error503)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(2);
});
it('should retry on 504 Gateway Timeout', async () => {
const error504 = new Error('Gateway Timeout');
error504.response = { status: 504 };
axios.get
.mockRejectedValueOnce(error504)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(2);
});
});
describe('retry behavior on network errors', () => {
it('should retry on network timeout (ECONNABORTED)', async () => {
const timeoutError = new Error('timeout of 30000ms exceeded');
timeoutError.code = 'ECONNABORTED';
// Network errors don't have a response property
axios.get
.mockRejectedValueOnce(timeoutError)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
const result = await promise;
expect(result).toBe('<html></html>');
expect(axios.get).toHaveBeenCalledTimes(2);
});
it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
const dnsError = new Error('getaddrinfo ENOTFOUND');
dnsError.code = 'ENOTFOUND';
axios.get
.mockRejectedValueOnce(dnsError)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(2);
});
it('should retry on connection refused (ECONNREFUSED)', async () => {
const connError = new Error('connect ECONNREFUSED');
connError.code = 'ECONNREFUSED';
axios.get
.mockRejectedValueOnce(connError)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(2);
});
});
describe('no retry on 4xx errors', () => {
it('should NOT retry on 400 Bad Request', async () => {
const error400 = new Error('Bad Request');
error400.response = { status: 400 };
axios.get.mockRejectedValueOnce(error400);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow();
expect(axios.get).toHaveBeenCalledTimes(1);
});
it('should NOT retry on 401 Unauthorized', async () => {
const error401 = new Error('Unauthorized');
error401.response = { status: 401 };
axios.get.mockRejectedValueOnce(error401);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow();
expect(axios.get).toHaveBeenCalledTimes(1);
});
it('should NOT retry on 403 Forbidden', async () => {
const error403 = new Error('Forbidden');
error403.response = { status: 403 };
axios.get.mockRejectedValueOnce(error403);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow();
expect(axios.get).toHaveBeenCalledTimes(1);
});
it('should NOT retry on 404 Not Found', async () => {
const error404 = new Error('Not Found');
error404.response = { status: 404 };
axios.get.mockRejectedValueOnce(error404);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow();
expect(axios.get).toHaveBeenCalledTimes(1);
});
it('should NOT retry on 429 Too Many Requests', async () => {
const error429 = new Error('Too Many Requests');
error429.response = { status: 429 };
axios.get.mockRejectedValueOnce(error429);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow();
expect(axios.get).toHaveBeenCalledTimes(1);
});
});
describe('exponential backoff', () => {
it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
// Fail 3 times, then succeed
axios.get
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
// First retry: 1 second delay
await jest.advanceTimersByTimeAsync(1000);
// Second retry: 2 second delay
await jest.advanceTimersByTimeAsync(2000);
// Third retry: 4 second delay
await jest.advanceTimersByTimeAsync(4000);
await promise;
expect(axios.get).toHaveBeenCalledTimes(4);
});
it('should log each retry attempt with attempt number', async () => {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
axios.get
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await jest.advanceTimersByTimeAsync(2000);
await promise;
// Check that attempt numbers were logged
expect(mockLogger.info).toHaveBeenCalledWith(
'Fetching page',
expect.objectContaining({ attempt: 1 })
);
expect(mockLogger.info).toHaveBeenCalledWith(
'Fetching page',
expect.objectContaining({ attempt: 2 })
);
expect(mockLogger.info).toHaveBeenCalledWith(
'Fetching page',
expect.objectContaining({ attempt: 3 })
);
});
it('should log failed attempts with error details', async () => {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
axios.get
.mockRejectedValueOnce(error500)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(mockLogger.warn).toHaveBeenCalledWith(
'Fetch attempt failed',
expect.objectContaining({
attempt: 1,
statusCode: 500,
isRetryable: true
})
);
});
it('should log wait time before retry', async () => {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
axios.get
.mockRejectedValueOnce(error500)
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
const promise = fetchPage('https://example.com', mockLogger);
await jest.advanceTimersByTimeAsync(1000);
await promise;
expect(mockLogger.info).toHaveBeenCalledWith(
'Waiting before retry',
expect.objectContaining({ delay: 1000 })
);
});
});
describe('retry exhaustion', () => {
// Use real timers for exhaustion tests to avoid promise handling issues
beforeEach(() => {
jest.useRealTimers();
});
afterEach(() => {
jest.useFakeTimers();
});
it('should throw error after all retries exhausted (4 attempts total)', async () => {
// Override config for faster tests
const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
try {
const error500 = new Error('Internal Server Error');
error500.response = { status: 500 };
// All 4 attempts fail (1 initial + 3 retries)
axios.get
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500)
.mockRejectedValueOnce(error500);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toThrow('Internal Server Error');
expect(axios.get).toHaveBeenCalledTimes(4);
} finally {
config.RETRY_CONFIG.baseDelay = originalBaseDelay;
}
});
it('should throw the last error when retries are exhausted', async () => {
// Override config for faster tests
const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
try {
const error500 = new Error('Server Error');
error500.response = { status: 500 };
axios.get.mockRejectedValue(error500);
await expect(fetchPage('https://example.com', mockLogger))
.rejects.toBe(error500);
} finally {
config.RETRY_CONFIG.baseDelay = originalBaseDelay;
}
});
});
describe('axios configuration', () => {
it('should set maxRedirects to 5', async () => {
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage('https://example.com', mockLogger);
expect(axios.get).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
maxRedirects: 5
})
);
});
it('should accept 2xx and 3xx status codes via validateStatus', async () => {
axios.get.mockResolvedValueOnce({
status: 200,
data: '<html></html>'
});
await fetchPage('https://example.com', mockLogger);
const callArgs = axios.get.mock.calls[0][1];
const validateStatus = callArgs.validateStatus;
// Should accept 2xx
expect(validateStatus(200)).toBe(true);
expect(validateStatus(201)).toBe(true);
expect(validateStatus(204)).toBe(true);
// Should accept 3xx
expect(validateStatus(301)).toBe(true);
expect(validateStatus(302)).toBe(true);
// Should reject 4xx and 5xx
expect(validateStatus(400)).toBe(false);
expect(validateStatus(404)).toBe(false);
expect(validateStatus(500)).toBe(false);
});
});
});

64
__tests__/scraper/fixtures/sample-listing-call.html
View File

@ -1,64 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - Call for Pricing</title></head>
<body>
<section class="spaces__tab-unit">
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9001"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="750"
data-spaces-sort-bed="1"
data-spaces-unit="P-101"
data-spaces-unit-id="9001"
data-spaces-unit-floor="1"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="500"
data-spaces-sort-plan-name="Penthouse A"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=9001"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
aria-label="Unit P-101">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-101</span>
</div>
</article>
<article
class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="9002"
data-spaces-soonest="Now"
data-spaces-sort-date="1706745600"
data-spaces-sort-price="Call for pricing"
data-spaces-sort-area="900"
data-spaces-sort-bed="2"
data-spaces-unit="P-102"
data-spaces-unit-id="9002"
data-spaces-unit-floor="2"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="501"
data-spaces-sort-plan-name="Penthouse B"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=9002"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
aria-label="Unit P-102">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">Call for pricing</span>
<span class="spaces-unit__unit">P-102</span>
</div>
</article>
</section>
</body>
</html>

9
__tests__/scraper/fixtures/sample-listing-empty.html
View File

@ -1,9 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - No Units</title></head>
<body>
<section class="spaces__tab-unit">
<!-- Empty - no units available -->
</section>
</body>
</html>

288
__tests__/scraper/fixtures/sample-listing.html
View File

@ -1,288 +0,0 @@
<!DOCTYPE html>
<html>
<head><title>Country Club Towers - Apartments</title></head>
<body>
<section class="spaces__tab-unit">
<article
class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154842"
data-spaces-soonest="2025-10-11"
data-spaces-sort-date="1760140800"
data-spaces-sort-price="5230"
data-spaces-sort-area="1210"
data-spaces-sort-bed="2"
data-spaces-unit="W2707"
data-spaces-unit-id="154842"
data-spaces-unit-floor="2755"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="10270"
data-spaces-sort-plan-name="Pyramid Peak - Terrace"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154842"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
aria-label="Unit W2707">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">5230</span>
<span class="spaces-unit__unit">W2707</span>
</div>
</article>
<article
class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154451"
data-spaces-soonest="2026-03-08"
data-spaces-sort-date="1772928000"
data-spaces-sort-price="2767"
data-spaces-sort-area="806"
data-spaces-sort-bed="1"
data-spaces-unit="E0901"
data-spaces-unit-id="154451"
data-spaces-unit-floor="2738"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8016"
data-spaces-sort-plan-name="Sunshine Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154451"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
aria-label="Unit E0901">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2767</span>
<span class="spaces-unit__unit">E0901</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154814"
data-spaces-soonest="2025-12-18"
data-spaces-sort-date="1766016000"
data-spaces-sort-price="2255"
data-spaces-sort-area="623"
data-spaces-sort-bed="0"
data-spaces-unit="W2603"
data-spaces-unit-id="154814"
data-spaces-unit-floor="2754"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8018"
data-spaces-sort-plan-name="Little Bear Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154814"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
aria-label="Unit W2603">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2255</span>
<span class="spaces-unit__unit">W2603</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154428"
data-spaces-soonest="2026-02-22"
data-spaces-sort-date="1771718400"
data-spaces-sort-price="4250"
data-spaces-sort-area="1152"
data-spaces-sort-bed="2"
data-spaces-unit="W0802"
data-spaces-unit-id="154428"
data-spaces-unit-floor="2737"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8017"
data-spaces-sort-plan-name="Mt. Princeton"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154428"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
aria-label="Unit W0802">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">4250</span>
<span class="spaces-unit__unit">W0802</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154778"
data-spaces-soonest="2025-12-31"
data-spaces-sort-date="1767139200"
data-spaces-sort-price="2495"
data-spaces-sort-area="764"
data-spaces-sort-bed="1"
data-spaces-unit="W2405"
data-spaces-unit-id="154778"
data-spaces-unit-floor="2752"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8020"
data-spaces-sort-plan-name="Grays Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154778"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
aria-label="Unit W2405">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W2405</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154442"
data-spaces-soonest="2025-02-01"
data-spaces-sort-date="1738368000"
data-spaces-sort-price="2495"
data-spaces-sort-area="802"
data-spaces-sort-bed="1"
data-spaces-unit="W0809"
data-spaces-unit-id="154442"
data-spaces-unit-floor="2737"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8022"
data-spaces-sort-plan-name="Pikes Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154442"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
aria-label="Unit W0809">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2495</span>
<span class="spaces-unit__unit">W0809</span>
</div>
</article>
<article
class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154764"
data-spaces-soonest="2026-02-17"
data-spaces-sort-date="1771286400"
data-spaces-sort-price="2435"
data-spaces-sort-area="715"
data-spaces-sort-bed="1"
data-spaces-unit="W2308"
data-spaces-unit-id="154764"
data-spaces-unit-floor="2751"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8023"
data-spaces-sort-plan-name="Longs Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154764"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
aria-label="Unit W2308">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2435</span>
<span class="spaces-unit__unit">W2308</span>
</div>
</article>
<article
class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
data-spaces-id="154612"
data-spaces-soonest="2026-03-24"
data-spaces-sort-date="1774310400"
data-spaces-sort-price="2683"
data-spaces-sort-area="797"
data-spaces-sort-bed="1"
data-spaces-unit="W1610"
data-spaces-unit-id="154612"
data-spaces-unit-floor="2744"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8024"
data-spaces-sort-plan-name="Torreys Peak"
data-spaces-sort-bath="1"
data-spaces-href="?spaces_tab=unit-detail&detail=154612"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
aria-label="Unit W1610">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">2683</span>
<span class="spaces-unit__unit">W1610</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
data-spaces-id="154495"
data-spaces-soonest="2025-11-07"
data-spaces-sort-date="1762473600"
data-spaces-sort-price="4470"
data-spaces-sort-area="1230"
data-spaces-sort-bed="2"
data-spaces-unit="E1011"
data-spaces-unit-id="154495"
data-spaces-unit-floor="2739"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8025"
data-spaces-sort-plan-name="Maroon Peak"
data-spaces-sort-bath="2"
data-spaces-href="?spaces_tab=unit-detail&detail=154495"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
aria-label="Unit E1011">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">4470</span>
<span class="spaces-unit__unit">E1011</span>
</div>
</article>
<article
class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
data-spaces-id="154919"
data-spaces-soonest="2025-10-27"
data-spaces-sort-date="1761523200"
data-spaces-sort-price="8581"
data-spaces-sort-area="1532"
data-spaces-sort-bed="2"
data-spaces-unit="E3205"
data-spaces-unit-id="154919"
data-spaces-unit-floor="2760"
data-spaces-obj="unit"
data-spaces-unavailable="false"
data-spaces-available="true"
data-spaces-community="Country Club Towers"
data-spaces-asset="420"
data-spaces-specials-content=""
data-spaces-plan-id="8037"
data-spaces-sort-plan-name="Mt. Antero"
data-spaces-sort-bath="2.5"
data-spaces-href="?spaces_tab=unit-detail&detail=154919"
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
aria-label="Unit E3205">
<div class="spaces-unit__inner">
<span class="spaces-unit__price">8581</span>
<span class="spaces-unit__unit">E3205</span>
</div>
</article>
</section>
</body>
</html>

582
__tests__/scraper/insertPrices.test.js
View File

@ -1,582 +0,0 @@
/**
* Tests for insertPrices() bulk operation
*
* Covers:
* - Only units with non-null prices are inserted
* - bulkWrite uses updateOne with upsert for idempotency
* - Filter uses unit_code + date_checked composite key
* - Price record includes unit_code, date_checked, price, last_updated, data_source
* - Empty price records logs warning and returns early
* - Error handling when bulkWrite fails
* - Logging of inserted and updated counts
* - Re-running on same day updates existing records (idempotent)
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
// Will require insertPrices after implementation
let insertPrices;
let mongoServer;
let client;
let db;
const PRICES_COLLECTION = 'unit_prices_scraper';
// Mock logger for capturing log calls
function createMockLogger() {
return {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
}
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
({ insertPrices } = require('../../services/scraperService'));
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean the prices collection before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
});
describe('insertPrices', () => {
const TODAY = '2026-02-05';
// ---------------------------------------------------------------
// 1. Only units with non-null prices are inserted
// ---------------------------------------------------------------
describe('filtering units with null prices', () => {
it('should only insert records for units with non-null prices', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'A101', price: 1200 },
{ unit_code: 'A102', price: null },
{ unit_code: 'A103', price: 1500 },
{ unit_code: 'A104', price: null }
];
await insertPrices(db, units, TODAY, logger);
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(2);
const unitCodes = docs.map(d => d.unit_code).sort();
expect(unitCodes).toEqual(['A101', 'A103']);
});
it('should skip units with price of null and not create any record for them', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'B101', price: null },
{ unit_code: 'B102', price: null }
];
const result = await insertPrices(db, units, TODAY, logger);
expect(result.insertedCount).toBe(0);
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(0);
});
it('should handle a mix of priced and null-priced units', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'C101', price: 950 },
{ unit_code: 'C102', price: null },
{ unit_code: 'C103', price: 1100 },
{ unit_code: 'C104', price: null },
{ unit_code: 'C105', price: 1300 }
];
await insertPrices(db, units, TODAY, logger);
const count = await db.collection(PRICES_COLLECTION).countDocuments();
expect(count).toBe(3);
});
});
// ---------------------------------------------------------------
// 2. bulkWrite uses updateOne with upsert for idempotency
// ---------------------------------------------------------------
describe('bulkWrite with upsert operations', () => {
it('should create updateOne operations for each priced unit', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'D101', price: 1200 },
{ unit_code: 'D102', price: 1500 }
];
const result = await insertPrices(db, units, TODAY, logger);
// Both units should be upserted (new inserts)
expect(result.insertedCount).toBeGreaterThan(0);
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(2);
});
it('should use upsert so re-running does not create duplicates', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'D201', price: 1200 }];
// First run
await insertPrices(db, units, TODAY, logger);
// Second run on same day
await insertPrices(db, units, TODAY, logger);
// Should still have only 1 document (not 2)
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(1);
});
});
// ---------------------------------------------------------------
// 3. Filter uses unit_code + date_checked composite key
// ---------------------------------------------------------------
describe('composite key: unit_code + date_checked', () => {
it('should use unit_code and date_checked as the filter for upsert', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'E101', price: 1200 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({
unit_code: 'E101',
date_checked: TODAY
});
expect(doc).not.toBeNull();
expect(doc.unit_code).toBe('E101');
expect(doc.date_checked).toBe(TODAY);
});
it('should create separate records for the same unit on different days', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'E201', price: 1200 }];
await insertPrices(db, units, '2026-02-04', logger);
await insertPrices(db, units, '2026-02-05', logger);
const docs = await db.collection(PRICES_COLLECTION)
.find({ unit_code: 'E201' })
.sort({ date_checked: 1 })
.toArray();
expect(docs).toHaveLength(2);
expect(docs[0].date_checked).toBe('2026-02-04');
expect(docs[1].date_checked).toBe('2026-02-05');
});
it('should create separate records for different units on the same day', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'E301', price: 1000 },
{ unit_code: 'E302', price: 1100 }
];
await insertPrices(db, units, TODAY, logger);
const docs = await db.collection(PRICES_COLLECTION)
.find({ date_checked: TODAY })
.toArray();
expect(docs).toHaveLength(2);
});
});
// ---------------------------------------------------------------
// 4. Price record includes required fields
// ---------------------------------------------------------------
describe('price record fields', () => {
it('should include unit_code in the price record', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'F101', price: 1400 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc.unit_code).toBe('F101');
});
it('should include date_checked in the price record', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'F201', price: 1400 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc.date_checked).toBe(TODAY);
});
it('should include price as a number in the price record', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'F301', price: 1550 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc.price).toBe(1550);
expect(typeof doc.price).toBe('number');
});
it('should include last_updated as an ISO timestamp string', async () => {
const logger = createMockLogger();
const beforeTime = new Date().toISOString();
const units = [{ unit_code: 'F401', price: 1600 }];
await insertPrices(db, units, TODAY, logger);
const afterTime = new Date().toISOString();
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc.last_updated).toBeDefined();
expect(typeof doc.last_updated).toBe('string');
expect(doc.last_updated >= beforeTime).toBe(true);
expect(doc.last_updated <= afterTime).toBe(true);
});
it('should include data_source set to "web_scraper"', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'F501', price: 1700 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc.data_source).toBe('web_scraper');
});
it('should include all five required fields in every price record', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'F601', price: 1800 }];
await insertPrices(db, units, TODAY, logger);
const doc = await db.collection(PRICES_COLLECTION).findOne({});
expect(doc).toHaveProperty('unit_code');
expect(doc).toHaveProperty('date_checked');
expect(doc).toHaveProperty('price');
expect(doc).toHaveProperty('last_updated');
expect(doc).toHaveProperty('data_source');
});
});
// ---------------------------------------------------------------
// 5. Empty price records logs warning and returns early
// ---------------------------------------------------------------
describe('empty price records', () => {
it('should log a warning when no units have prices', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'G101', price: null },
{ unit_code: 'G102', price: null }
];
await insertPrices(db, units, TODAY, logger);
expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
});
it('should return insertedCount of 0 for all null-priced units', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'G201', price: null }];
const result = await insertPrices(db, units, TODAY, logger);
expect(result.insertedCount).toBe(0);
});
it('should log a warning when units array is empty', async () => {
const logger = createMockLogger();
await insertPrices(db, [], TODAY, logger);
expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
});
it('should not perform any database writes for empty/null-priced arrays', async () => {
const logger = createMockLogger();
await insertPrices(db, [], TODAY, logger);
const count = await db.collection(PRICES_COLLECTION).countDocuments();
expect(count).toBe(0);
});
});
// ---------------------------------------------------------------
// 6. Error handling when bulkWrite fails
// ---------------------------------------------------------------
describe('error handling', () => {
it('should throw error when bulkWrite fails', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'H101', price: 1000 }];
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
});
it('should log error details when bulkWrite fails', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'H201', price: 1000 }];
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
try {
await insertPrices(mockDb, units, TODAY, logger);
} catch (e) {
// Expected to throw
}
expect(logger.error).toHaveBeenCalledWith(
'Failed to insert prices',
expect.objectContaining({
errorType: 'Error',
errorMessage: 'Write concern timeout'
})
);
});
it('should re-throw the original error', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'H301', price: 1000 }];
const originalError = new TypeError('Invalid operation');
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(originalError)
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
});
});
// ---------------------------------------------------------------
// 7. Logging of inserted and updated counts
// ---------------------------------------------------------------
describe('logging of result counts', () => {
it('should log inserted and updated counts on success', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'I101', price: 1000 },
{ unit_code: 'I102', price: 1100 }
];
await insertPrices(db, units, TODAY, logger);
expect(logger.info).toHaveBeenCalledWith(
'Prices inserted',
expect.objectContaining({
inserted: expect.any(Number),
updated: expect.any(Number)
})
);
});
it('should log correct counts for new inserts', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'I201', price: 1000 },
{ unit_code: 'I202', price: 1100 }
];
await insertPrices(db, units, TODAY, logger);
// For new inserts: inserted=2, updated=0
expect(logger.info).toHaveBeenCalledWith(
'Prices inserted',
expect.objectContaining({
inserted: 2,
updated: 0
})
);
});
it('should log correct counts when updating existing records', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'I301', price: 1000 }];
// First insert
await insertPrices(db, units, TODAY, logger);
// Reset mock
logger.info.mockClear();
// Second run updates existing
const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
await insertPrices(db, updatedUnits, TODAY, logger);
expect(logger.info).toHaveBeenCalledWith(
'Prices inserted',
expect.objectContaining({
inserted: 0,
updated: 1
})
);
});
});
// ---------------------------------------------------------------
// 8. Re-running on same day updates existing records (idempotent)
// ---------------------------------------------------------------
describe('idempotent re-runs on same day', () => {
it('should update the price when re-running on the same day', async () => {
const logger = createMockLogger();
// First run with original price
await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
const doc1 = await db.collection(PRICES_COLLECTION).findOne({
unit_code: 'J101',
date_checked: TODAY
});
expect(doc1.price).toBe(1200);
// Re-run with updated price (same day)
await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
const doc2 = await db.collection(PRICES_COLLECTION).findOne({
unit_code: 'J101',
date_checked: TODAY
});
expect(doc2.price).toBe(1250);
// Should still have only 1 record
const count = await db.collection(PRICES_COLLECTION).countDocuments({
unit_code: 'J101',
date_checked: TODAY
});
expect(count).toBe(1);
});
it('should update last_updated when re-running on the same day', async () => {
const logger = createMockLogger();
// First run
await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
const firstUpdated = doc1.last_updated;
// Small delay to ensure different timestamp
await new Promise(resolve => setTimeout(resolve, 10));
// Re-run on same day
await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
const secondUpdated = doc2.last_updated;
expect(secondUpdated > firstUpdated).toBe(true);
});
it('should return combined insert/update count reflecting the operation', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'J301', price: 1000 },
{ unit_code: 'J302', price: 1100 }
];
// First run - inserts
const result1 = await insertPrices(db, units, TODAY, logger);
expect(result1.insertedCount).toBe(2);
// Re-run - updates
const result2 = await insertPrices(db, units, TODAY, logger);
expect(result2.insertedCount).toBe(2); // combined upserted + modified
// Still only 2 total records
const count = await db.collection(PRICES_COLLECTION).countDocuments();
expect(count).toBe(2);
});
it('should handle a mix of new and existing records on re-run', async () => {
const logger = createMockLogger();
// First run with unit A
await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
// Re-run with unit A (existing) and unit B (new)
const mixedUnits = [
{ unit_code: 'J401', price: 1050 },
{ unit_code: 'J402', price: 1200 }
];
await insertPrices(db, mixedUnits, TODAY, logger);
// Should have 2 total records
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(2);
// Verify updated price
const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
expect(unitA.price).toBe(1050);
});
});
// ---------------------------------------------------------------
// 9. Uses ordered: false for performance
// ---------------------------------------------------------------
describe('ordered: false for performance', () => {
it('should successfully process all priced units in a batch', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'K101', price: 900 },
{ unit_code: 'K102', price: 1000 },
{ unit_code: 'K103', price: 1100 },
{ unit_code: 'K104', price: 1200 },
{ unit_code: 'K105', price: 1300 }
];
await insertPrices(db, units, TODAY, logger);
const count = await db.collection(PRICES_COLLECTION).countDocuments();
expect(count).toBe(5);
});
});
// ---------------------------------------------------------------
// 10. Uses correct collection
// ---------------------------------------------------------------
describe('collection usage', () => {
it('should write to the configured PRICES collection', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'L101', price: 1400 }];
await insertPrices(db, units, TODAY, logger);
// Verify data is in the correct collection
const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
expect(doc).not.toBeNull();
});
});
});

449
__tests__/scraper/markStaleUnits.test.js
View File

@ -1,449 +0,0 @@
/**
* Tests for markStaleUnits() availability tracking
*
* Covers:
* - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
* - $set updates available to false and sets marked_stale_date
* - Units in currentCodes are NOT marked stale
* - Already unavailable units are not modified
* - Error handling when updateMany fails
* - Logging of modified count
* - Empty currentUnitCodes handles gracefully
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
// Will require markStaleUnits after implementation
let markStaleUnits;
let mongoServer;
let client;
let db;
const UNITS_COLLECTION = 'units_scraper';
// Mock logger for capturing log calls
function createMockLogger() {
return {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
}
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
({ markStaleUnits } = require('../../services/scraperService'));
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean collections before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
});
describe('markStaleUnits', () => {
// ---------------------------------------------------------------
// 1. updateMany is called with correct filter
// (unit_code $nin currentCodes AND available: true)
// ---------------------------------------------------------------
describe('filter criteria', () => {
it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
const logger = createMockLogger();
// Seed units in the database
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'A101', available: true, price: 1200 },
{ unit_code: 'A102', available: true, price: 1300 },
{ unit_code: 'B201', available: true, price: 1500 }
]);
// Current scrape found A101 and A102, but NOT B201
const currentCodes = new Set(['A101', 'A102']);
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
// B201 should be marked stale (it was available but not in current scrape)
expect(result.modifiedCount).toBe(1);
// Verify B201 was updated
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
expect(staleUnit.available).toBe(false);
expect(staleUnit.marked_stale_date).toBe(date);
});
it('should only affect units NOT in the currentUnitCodes set', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'X1', available: true },
{ unit_code: 'X2', available: true },
{ unit_code: 'X3', available: true },
{ unit_code: 'X4', available: true }
]);
// Current scrape found X1 and X3
const currentCodes = new Set(['X1', 'X3']);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
// X1 and X3 should still be available
const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
expect(x1.available).toBe(true);
expect(x3.available).toBe(true);
// X2 and X4 should be marked stale
const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
expect(x2.available).toBe(false);
expect(x4.available).toBe(false);
});
});
// ---------------------------------------------------------------
// 2. $set updates available to false and sets marked_stale_date
// ---------------------------------------------------------------
describe('$set fields', () => {
it('should set available to false for stale units', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'S100', available: true, price: 1000 },
{ unit_code: 'S200', available: true, price: 1100 }
]);
const currentCodes = new Set(['S100']);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
expect(staleUnit.available).toBe(false);
});
it('should set marked_stale_date to the provided date', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'D100', available: true }
]);
const currentCodes = new Set([]);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
expect(doc.marked_stale_date).toBe('2026-02-05');
});
it('should update both available and marked_stale_date in the same operation', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'C100', available: true, price: 900 }
]);
const currentCodes = new Set([]);
const date = '2026-01-15';
await markStaleUnits(db, currentCodes, date, logger);
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
expect(doc.available).toBe(false);
expect(doc.marked_stale_date).toBe('2026-01-15');
});
});
// ---------------------------------------------------------------
// 3. Units in currentCodes are NOT marked stale
// ---------------------------------------------------------------
describe('units in currentCodes not affected', () => {
it('should not modify units that are in the currentUnitCodes set', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'KEEP-1', available: true, price: 1200 },
{ unit_code: 'KEEP-2', available: true, price: 1300 },
{ unit_code: 'STALE-1', available: true, price: 1400 }
]);
const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
// Kept units should retain their original state
const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
expect(keep1.available).toBe(true);
expect(keep1.marked_stale_date).toBeUndefined();
expect(keep2.available).toBe(true);
expect(keep2.marked_stale_date).toBeUndefined();
});
it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'ALL-1', available: true },
{ unit_code: 'ALL-2', available: true }
]);
const currentCodes = new Set(['ALL-1', 'ALL-2']);
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
expect(result.modifiedCount).toBe(0);
});
});
// ---------------------------------------------------------------
// 4. Already unavailable units are not modified
// ---------------------------------------------------------------
describe('already unavailable units', () => {
it('should not modify units that are already unavailable (available: false)', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
{ unit_code: 'UNAVAIL-2', available: false },
{ unit_code: 'AVAIL-1', available: true }
]);
// None of these units are in the current scrape
const currentCodes = new Set([]);
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
// Only AVAIL-1 should be modified (the already-unavailable ones should not be)
expect(result.modifiedCount).toBe(1);
// Verify the already-unavailable unit's date was NOT changed
const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
expect(unavail1.marked_stale_date).toBe('2026-01-01');
});
it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'U1', available: false },
{ unit_code: 'U2', available: false }
]);
const currentCodes = new Set([]);
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
expect(result.modifiedCount).toBe(0);
});
});
// ---------------------------------------------------------------
// 5. Error handling when updateMany fails
// ---------------------------------------------------------------
describe('error handling', () => {
it('should throw error when updateMany fails', async () => {
const logger = createMockLogger();
const mockCollection = {
updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
const currentCodes = new Set(['A1']);
const date = '2026-02-05';
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
});
it('should log error details when updateMany fails', async () => {
const logger = createMockLogger();
const mockCollection = {
updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
const currentCodes = new Set(['A1']);
const date = '2026-02-05';
try {
await markStaleUnits(mockDb, currentCodes, date, logger);
} catch (e) {
// Expected to throw
}
expect(logger.error).toHaveBeenCalledWith(
'Failed to mark stale units',
expect.objectContaining({
errorType: 'Error',
errorMessage: 'Write concern timeout'
})
);
});
it('should re-throw the original error', async () => {
const logger = createMockLogger();
const originalError = new TypeError('Invalid operation');
const mockCollection = {
updateMany: jest.fn().mockRejectedValue(originalError)
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
const currentCodes = new Set(['A1']);
const date = '2026-02-05';
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
});
});
// ---------------------------------------------------------------
// 6. Logging of modified count
// ---------------------------------------------------------------
describe('logging', () => {
it('should log the count of stale units marked', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'LOG-1', available: true },
{ unit_code: 'LOG-2', available: true },
{ unit_code: 'LOG-3', available: true }
]);
const currentCodes = new Set(['LOG-1']);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
expect(logger.info).toHaveBeenCalledWith(
'Stale units marked',
expect.objectContaining({ count: 2 })
);
});
it('should log count of 0 when no units are marked stale', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'ZERO-1', available: true }
]);
const currentCodes = new Set(['ZERO-1']);
const date = '2026-02-05';
await markStaleUnits(db, currentCodes, date, logger);
expect(logger.info).toHaveBeenCalledWith(
'Stale units marked',
expect.objectContaining({ count: 0 })
);
});
});
// ---------------------------------------------------------------
// 7. Empty currentUnitCodes handles gracefully
// ---------------------------------------------------------------
describe('empty currentUnitCodes', () => {
it('should mark all available units as stale when currentUnitCodes is empty', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'EMPTY-1', available: true },
{ unit_code: 'EMPTY-2', available: true },
{ unit_code: 'EMPTY-3', available: true }
]);
const currentCodes = new Set();
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
expect(result.modifiedCount).toBe(3);
// All should be marked unavailable
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
for (const doc of docs) {
expect(doc.available).toBe(false);
expect(doc.marked_stale_date).toBe('2026-02-05');
}
});
it('should not throw when currentUnitCodes is an empty Set', async () => {
const logger = createMockLogger();
const currentCodes = new Set();
const date = '2026-02-05';
// Should not throw even with empty collection and empty set
await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
});
it('should handle empty database with empty currentUnitCodes', async () => {
const logger = createMockLogger();
const currentCodes = new Set();
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
expect(result.modifiedCount).toBe(0);
expect(logger.info).toHaveBeenCalledWith(
'Stale units marked',
expect.objectContaining({ count: 0 })
);
});
});
// ---------------------------------------------------------------
// 8. Return value
// ---------------------------------------------------------------
describe('return value', () => {
it('should return the MongoDB updateMany result', async () => {
const logger = createMockLogger();
await db.collection(UNITS_COLLECTION).insertMany([
{ unit_code: 'RET-1', available: true },
{ unit_code: 'RET-2', available: true }
]);
const currentCodes = new Set(['RET-1']);
const date = '2026-02-05';
const result = await markStaleUnits(db, currentCodes, date, logger);
// updateMany result should have these standard properties
expect(result).toHaveProperty('matchedCount');
expect(result).toHaveProperty('modifiedCount');
expect(result).toHaveProperty('acknowledged');
expect(result.acknowledged).toBe(true);
});
});
});

517
__tests__/scraper/parseUnits.test.js
View File

@ -1,517 +0,0 @@
/**
* Tests for parseUnits() - HTML parsing with cheerio
*
* Covers:
* - Extraction of all data-spaces-* attributes
* - Image URL extraction from nested img elements
* - Container detection (section.spaces__tab-unit)
* - Empty / missing container handling
* - Deduplication by unit_code
* - Logging behavior (info, warn, error)
* - Missing attribute handling
*/
const fs = require('fs');
const path = require('path');
const { parseUnits } = require('../../services/scraperService');
const fixturesDir = path.join(__dirname, 'fixtures');
// Helper: create a mock logger
function createMockLogger() {
return {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
}
describe('parseUnits', () => {
let sampleHtml;
let mockLogger;
beforeAll(() => {
sampleHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing.html'),
'utf-8'
);
});
beforeEach(() => {
mockLogger = createMockLogger();
});
// ---------------------------------------------------------------
// 1. Container detection
// ---------------------------------------------------------------
describe('container detection', () => {
it('should find section.spaces__tab-unit container in valid HTML', () => {
const units = parseUnits(sampleHtml, mockLogger);
// Should not log an error about missing container
expect(mockLogger.error).not.toHaveBeenCalled();
expect(units.length).toBeGreaterThan(0);
});
it('should return empty array when section container is missing', () => {
const html = '<html><body><div>No units here</div></body></html>';
const units = parseUnits(html, mockLogger);
expect(units).toEqual([]);
expect(mockLogger.error).toHaveBeenCalledWith(
expect.stringContaining('Container section.spaces__tab-unit not found'),
expect.any(Object)
);
});
it('should return empty array for empty HTML string', () => {
const units = parseUnits('', mockLogger);
expect(units).toEqual([]);
expect(mockLogger.error).toHaveBeenCalled();
});
it('should return empty array when container exists but has no articles', () => {
const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
const units = parseUnits(html, mockLogger);
expect(units).toEqual([]);
// Container found, no error about container
expect(mockLogger.error).not.toHaveBeenCalled();
});
});
// ---------------------------------------------------------------
// 2. Extraction of all data-spaces-* attributes
// ---------------------------------------------------------------
describe('attribute extraction', () => {
it('should extract all expected attributes from a fully populated article', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'W2707');
expect(unit1).toBeDefined();
// Core identifiers
expect(unit1.id).toBe('154842');
expect(unit1.unit_code).toBe('W2707');
expect(unit1.unit_id).toBe('154842');
// Physical attributes
expect(unit1.floor).toBe('2755');
expect(unit1.area).toBe('1210');
expect(unit1.bed_count).toBe('2');
expect(unit1.bath_count).toBe('2');
// Pricing
expect(unit1.price).toBe('5230');
// Availability
expect(unit1.available).toBe('true');
expect(unit1.unavailable).toBe('false');
expect(unit1.soonest).toBe('2025-10-11');
expect(unit1.date_available).toBe('1760140800');
// Plan information
expect(unit1.plan_id).toBe('10270');
expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
// Property information
expect(unit1.obj_type).toBe('unit');
expect(unit1.community).toBe('Country Club Towers');
expect(unit1.asset).toBe('420');
// URLs
expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
// Specials
expect(unit1.specials_content).toBe('');
});
it('should extract correct count of units from the fixture', () => {
const units = parseUnits(sampleHtml, mockLogger);
expect(units).toHaveLength(10);
});
it('should return raw string values without type conversion', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'W2707');
// All values should be strings (raw extraction, no conversion)
expect(typeof unit.id).toBe('string');
expect(typeof unit.price).toBe('string');
expect(typeof unit.floor).toBe('string');
expect(typeof unit.available).toBe('string');
expect(typeof unit.bed_count).toBe('string');
expect(typeof unit.bath_count).toBe('string');
});
});
// ---------------------------------------------------------------
// 3. Image URL extraction
// ---------------------------------------------------------------
describe('image URL extraction', () => {
it('should extract image_url from img src attribute', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
<img src="https://example.com/images/unit-1.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
});
it('should fall back to data-src when src is not present', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
<img data-src="https://example.com/images/unit-2.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
});
it('should not set image_url when no img element exists', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
<div>No image here</div>
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(units[0].image_url).toBeUndefined();
});
});
// ---------------------------------------------------------------
// 4. Missing attributes
// ---------------------------------------------------------------
describe('missing attributes', () => {
it('should return undefined for attributes not present on the article', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit).toBeDefined();
expect(unit.floor).toBeUndefined();
expect(unit.area).toBeUndefined();
expect(unit.bed_count).toBeUndefined();
expect(unit.bath_count).toBeUndefined();
expect(unit.soonest).toBeUndefined();
expect(unit.date_available).toBeUndefined();
expect(unit.plan_id).toBeUndefined();
expect(unit.plan_name).toBeUndefined();
expect(unit.href).toBeUndefined();
expect(unit.inventory_href).toBeUndefined();
expect(unit.specials_content).toBeUndefined();
});
it('should still extract the attributes that are present on a sparse article', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit.id).toBe('1005');
expect(unit.unit_code).toBe('SPARSE-1');
expect(unit.unit_id).toBe('5005');
expect(unit.price).toBe('1300');
expect(unit.available).toBe('true');
expect(unit.obj_type).toBe('unit');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('100');
});
});
// ---------------------------------------------------------------
// 5. Deduplication
// ---------------------------------------------------------------
describe('deduplication', () => {
it('should deduplicate units with the same unit_code', () => {
// Build HTML with duplicate articles
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const codes = units.map(u => u.unit_code);
// Should have 2 unique unit codes
expect(units).toHaveLength(2);
expect(new Set(codes).size).toBe(codes.length);
});
it('should keep the first occurrence when deduplicating', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
// First occurrence (id=1, price=1000) should be kept
expect(units[0].id).toBe('1');
expect(units[0].price).toBe('1000');
});
it('should log a warning when duplicates are removed', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
</section>
</body></html>
`;
parseUnits(html, mockLogger);
expect(mockLogger.warn).toHaveBeenCalledWith(
'Duplicate units removed',
expect.objectContaining({
original: 3,
deduplicated: 2
})
);
});
it('should not log a warning when there are no duplicates', () => {
parseUnits(sampleHtml, mockLogger);
// The fixture has no duplicates, so no warn about duplicates
const warnCalls = mockLogger.warn.mock.calls;
const duplicateWarns = warnCalls.filter(
call => call[0] === 'Duplicate units removed'
);
expect(duplicateWarns).toHaveLength(0);
});
it('should filter out articles with no unit_code during deduplication', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-sort-price="1000"></article>
<article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
// Article without unit_code should be filtered out
expect(units).toHaveLength(1);
expect(units[0].unit_code).toBe('VALID-1');
});
});
// ---------------------------------------------------------------
// 6. Logging behavior
// ---------------------------------------------------------------
describe('logging', () => {
it('should log the count of parsed units', () => {
parseUnits(sampleHtml, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 10 })
);
});
it('should log error when container is not found', () => {
const html = '<html><body><p>Nothing here</p></body></html>';
parseUnits(html, mockLogger);
expect(mockLogger.error).toHaveBeenCalledTimes(1);
expect(mockLogger.error).toHaveBeenCalledWith(
expect.stringContaining('Container section.spaces__tab-unit not found'),
expect.any(Object)
);
});
});
// ---------------------------------------------------------------
// 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
// ---------------------------------------------------------------
describe('fixture: sample-listing-empty.html', () => {
let emptyHtml;
beforeAll(() => {
emptyHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-empty.html'),
'utf-8'
);
});
it('should return empty array when fixture has container but no articles', () => {
const units = parseUnits(emptyHtml, mockLogger);
expect(units).toEqual([]);
// Container exists so no error about missing container
expect(mockLogger.error).not.toHaveBeenCalled();
});
it('should log zero parsed units for empty fixture', () => {
parseUnits(emptyHtml, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 0 })
);
});
});
describe('fixture: sample-listing-call.html', () => {
let callHtml;
beforeAll(() => {
callHtml = fs.readFileSync(
path.join(fixturesDir, 'sample-listing-call.html'),
'utf-8'
);
});
it('should extract both units from the call-for-pricing fixture', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units).toHaveLength(2);
expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
});
it('should preserve "Call for pricing" as raw string in the price field', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].price).toBe('Call for pricing');
expect(units[1].price).toBe('Call for pricing');
});
it('should extract all attributes correctly from call-for-pricing units', () => {
const units = parseUnits(callHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'P-101');
expect(unit.id).toBe('9001');
expect(unit.unit_id).toBe('9001');
expect(unit.floor).toBe('1');
expect(unit.area).toBe('750');
expect(unit.bed_count).toBe('1');
expect(unit.bath_count).toBe('1');
expect(unit.available).toBe('true');
expect(unit.unavailable).toBe('false');
expect(unit.soonest).toBe('Now');
expect(unit.date_available).toBe('1706745600');
expect(unit.plan_id).toBe('500');
expect(unit.plan_name).toBe('Penthouse A');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('420');
expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
expect(unit.specials_content).toBe('');
});
it('should not have image_url when fixture units lack img elements', () => {
const units = parseUnits(callHtml, mockLogger);
expect(units[0].image_url).toBeUndefined();
expect(units[1].image_url).toBeUndefined();
});
});
// ---------------------------------------------------------------
// 8. Multiple units with varied data
// ---------------------------------------------------------------
describe('varied unit data', () => {
it('should handle unavailable units correctly', () => {
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="4001"
data-spaces-unit="UNAVAIL-1"
data-spaces-sort-price="2000"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date="">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit.available).toBe('false');
expect(unit.unavailable).toBe('true');
expect(unit.soonest).toBe('');
expect(unit.date_available).toBe('');
});
it('should handle half bath counts', () => {
const units = parseUnits(sampleHtml, mockLogger);
// E3205 is the penthouse with 2.5 baths
const unit = units.find(u => u.unit_code === 'E3205');
expect(unit.bath_count).toBe('2.5');
});
it('should handle studio units (bed_count = 0)', () => {
const units = parseUnits(sampleHtml, mockLogger);
// W2603 is the studio with bed_count 0
const unit = units.find(u => u.unit_code === 'W2603');
expect(unit.bed_count).toBe('0');
});
});
});

307
__tests__/scraper/recordScraperRun.test.js
View File

@ -1,307 +0,0 @@
/**
* Tests for recordScraperRun() history persistence
*
* Covers:
* - insertOne is called with runData plus recordedAt timestamp
* - All runData fields are preserved (jobId, trigger, status, duration, etc.)
* - Error is caught and logged but NOT thrown (graceful failure)
* - Returns null on error instead of crashing
* - Returns insert result on success
* - recordedAt is a valid Date object
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
// Will require after implementation
let recordScraperRun;
let mongoServer;
let client;
let db;
let logger;
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
({ recordScraperRun } = require('../../services/scraperService'));
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean all collections before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
// Create fresh mock logger for each test
logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
});
describe('recordScraperRun', () => {
const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
// Sample runData matching the structure from runScrape()'s finally block
function createSampleRunData(overrides = {}) {
return {
jobId: 'test-job-001',
trigger: 'manual',
dryRun: false,
status: 'success',
startedAt: '2026-02-06T06:00:00.000Z',
completedAt: '2026-02-06T06:00:12.345Z',
duration: 12345,
unitsProcessed: 50,
pricesInserted: 48,
newUnitsCount: 3,
rentedUnitsCount: 1,
staleUnitsCount: 2,
errors: [],
...overrides
};
}
// ---------------------------------------------------------------
// 1. insertOne is called with runData plus recordedAt timestamp
// ---------------------------------------------------------------
describe('insertOne with runData and recordedAt', () => {
it('should insert a document into the scraper_runs collection', async () => {
const runData = createSampleRunData();
await recordScraperRun(db, runData, logger);
const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(1);
});
it('should include a recordedAt field in the inserted document', async () => {
const runData = createSampleRunData();
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.recordedAt).toBeDefined();
});
});
// ---------------------------------------------------------------
// 2. All runData fields are preserved
// ---------------------------------------------------------------
describe('preserving all runData fields', () => {
it('should preserve jobId, trigger, status, and duration', async () => {
const runData = createSampleRunData({
jobId: 'preserve-test-001',
trigger: 'scheduled',
status: 'failed',
duration: 9999
});
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.jobId).toBe('preserve-test-001');
expect(doc.trigger).toBe('scheduled');
expect(doc.status).toBe('failed');
expect(doc.duration).toBe(9999);
});
it('should preserve unit processing metrics', async () => {
const runData = createSampleRunData({
unitsProcessed: 42,
pricesInserted: 40,
newUnitsCount: 5,
rentedUnitsCount: 2,
staleUnitsCount: 3
});
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.unitsProcessed).toBe(42);
expect(doc.pricesInserted).toBe(40);
expect(doc.newUnitsCount).toBe(5);
expect(doc.rentedUnitsCount).toBe(2);
expect(doc.staleUnitsCount).toBe(3);
});
it('should preserve timing fields (startedAt, completedAt)', async () => {
const runData = createSampleRunData({
startedAt: '2026-02-06T10:00:00.000Z',
completedAt: '2026-02-06T10:00:15.500Z'
});
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
});
it('should preserve dryRun flag', async () => {
const runData = createSampleRunData({ dryRun: true });
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.dryRun).toBe(true);
});
it('should preserve errors array when it has entries', async () => {
const runData = createSampleRunData({
status: 'failed',
errors: ['Connection timeout', 'Retry exhausted']
});
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
});
it('should preserve empty errors array for successful runs', async () => {
const runData = createSampleRunData({ errors: [] });
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.errors).toEqual([]);
});
});
// ---------------------------------------------------------------
// 3. Error is caught and logged but NOT thrown (graceful failure)
// ---------------------------------------------------------------
describe('graceful error handling', () => {
it('should NOT throw when insertOne fails', async () => {
const runData = createSampleRunData();
// Create a mock db that throws on insertOne
const mockCollection = {
insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
// This should NOT throw
await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
});
it('should log error message when insert fails', async () => {
const runData = createSampleRunData();
const mockCollection = {
insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await recordScraperRun(mockDb, runData, logger);
expect(logger.error).toHaveBeenCalledWith(
'Failed to record scraper run',
{ errorMessage: 'Disk full' }
);
});
});
// ---------------------------------------------------------------
// 4. Returns null on error instead of crashing
// ---------------------------------------------------------------
describe('return value on error', () => {
it('should return null when insertOne throws', async () => {
const runData = createSampleRunData();
const mockCollection = {
insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
const result = await recordScraperRun(mockDb, runData, logger);
expect(result).toBeNull();
});
});
// ---------------------------------------------------------------
// 5. Returns insert result on success
// ---------------------------------------------------------------
describe('return value on success', () => {
it('should return the insertOne result object', async () => {
const runData = createSampleRunData();
const result = await recordScraperRun(db, runData, logger);
expect(result).toBeDefined();
expect(result).not.toBeNull();
});
it('should return result with acknowledged property', async () => {
const runData = createSampleRunData();
const result = await recordScraperRun(db, runData, logger);
expect(result.acknowledged).toBe(true);
});
it('should return result with insertedId', async () => {
const runData = createSampleRunData();
const result = await recordScraperRun(db, runData, logger);
expect(result.insertedId).toBeDefined();
});
});
// ---------------------------------------------------------------
// 6. recordedAt is a valid Date object
// ---------------------------------------------------------------
describe('recordedAt field', () => {
it('should set recordedAt as a Date instance', async () => {
const runData = createSampleRunData();
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.recordedAt).toBeInstanceOf(Date);
});
it('should set recordedAt close to current time', async () => {
const beforeTime = new Date();
const runData = createSampleRunData();
await recordScraperRun(db, runData, logger);
const afterTime = new Date();
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
});
it('should not overwrite any existing runData fields with recordedAt', async () => {
const runData = createSampleRunData();
await recordScraperRun(db, runData, logger);
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
// Verify the original fields still exist alongside recordedAt
expect(doc.jobId).toBe(runData.jobId);
expect(doc.status).toBe(runData.status);
expect(doc.recordedAt).toBeInstanceOf(Date);
});
});
});

192
__tests__/scraper/scraperConfig.test.js
View File

@ -1,192 +0,0 @@
/**
* Tests for config/scraper.js
* Tests scraper configuration constants and environment variable overrides
*/
describe('config/scraper', () => {
// Store original env values to restore after tests
const originalEnv = { ...process.env };
beforeEach(() => {
// Clear all scraper-related env vars before each test
delete process.env.SCRAPER_SCHEDULE;
delete process.env.SCRAPER_TIMEZONE;
delete process.env.SCRAPER_ENABLED;
delete process.env.SCRAPER_TIMEOUT;
delete process.env.SCRAPER_UNITS_COLLECTION;
delete process.env.SCRAPER_PRICES_COLLECTION;
delete process.env.SCRAPER_SUMMARIES_COLLECTION;
delete process.env.SCRAPER_RUNS_COLLECTION;
// Clear the require cache to reload config with fresh env
jest.resetModules();
});
afterAll(() => {
// Restore original environment
process.env = originalEnv;
});
describe('TARGET_URL', () => {
test('should be set to the correct apartment listing URL', () => {
const config = require('../../config/scraper');
expect(config.TARGET_URL).toBe(
'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
);
});
});
describe('SCRAPER_SCHEDULE', () => {
test('should default to "0 6 * * *" when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
});
test('should use env var override when set', () => {
process.env.SCRAPER_SCHEDULE = '0 12 * * *';
const config = require('../../config/scraper');
expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
});
});
describe('SCRAPER_TIMEZONE', () => {
test('should default to "America/Denver" when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
});
test('should use env var override when set', () => {
process.env.SCRAPER_TIMEZONE = 'America/New_York';
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/New_York');
});
});
describe('SCRAPER_ENABLED', () => {
test('should default to true when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_ENABLED).toBe(true);
});
test('should be false when env var is set to "false"', () => {
process.env.SCRAPER_ENABLED = 'false';
const config = require('../../config/scraper');
expect(config.SCRAPER_ENABLED).toBe(false);
});
test('should be true when env var is set to any other value', () => {
process.env.SCRAPER_ENABLED = 'true';
const config = require('../../config/scraper');
expect(config.SCRAPER_ENABLED).toBe(true);
});
test('should be true when env var is empty string', () => {
process.env.SCRAPER_ENABLED = '';
const config = require('../../config/scraper');
expect(config.SCRAPER_ENABLED).toBe(true);
});
});
describe('SCRAPER_TIMEOUT', () => {
test('should default to 30000 when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEOUT).toBe(30000);
});
test('should use env var override when set', () => {
process.env.SCRAPER_TIMEOUT = '60000';
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEOUT).toBe(60000);
});
test('should parse string to integer', () => {
process.env.SCRAPER_TIMEOUT = '45000';
const config = require('../../config/scraper');
expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
expect(config.SCRAPER_TIMEOUT).toBe(45000);
});
});
describe('USER_AGENT', () => {
test('should be a valid User-Agent string', () => {
const config = require('../../config/scraper');
expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
});
});
describe('RETRY_CONFIG', () => {
test('should have maxRetries property', () => {
const config = require('../../config/scraper');
expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
expect(config.RETRY_CONFIG.maxRetries).toBe(3);
});
test('should have baseDelay property', () => {
const config = require('../../config/scraper');
expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
});
test('should have timeout property', () => {
const config = require('../../config/scraper');
expect(config.RETRY_CONFIG).toHaveProperty('timeout');
expect(config.RETRY_CONFIG.timeout).toBe(30000);
});
test('timeout should respect SCRAPER_TIMEOUT env var', () => {
process.env.SCRAPER_TIMEOUT = '45000';
const config = require('../../config/scraper');
expect(config.RETRY_CONFIG.timeout).toBe(45000);
});
});
describe('COLLECTIONS', () => {
describe('default values', () => {
test('UNITS should default to "units_scraper"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
});
test('PRICES should default to "unit_prices_scraper"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
});
test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
});
test('SCRAPER_RUNS should default to "scraper_runs"', () => {
const config = require('../../config/scraper');
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
});
});
describe('env var overrides', () => {
test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
const config = require('../../config/scraper');
expect(config.COLLECTIONS.UNITS).toBe('units_custom');
});
test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
const config = require('../../config/scraper');
expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
});
test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
const config = require('../../config/scraper');
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
});
test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
const config = require('../../config/scraper');
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
});
});
});
});

1590
__tests__/scraper/scraperJob.test.js
View File

File diff suppressed because it is too large Load Diff

1740
__tests__/scraper/scraperRoutes.test.js
View File

File diff suppressed because it is too large Load Diff

739
__tests__/scraper/scraperService.test.js
View File

@ -1,739 +0,0 @@
/**
* Tests for runScrape() orchestration function
*
* Covers:
* - Full workflow completes with mocked dependencies
* - Returns correct result structure with jobId, status, metrics
* - Handles dryRun option (skip DB writes)
* - Handles htmlContent option (use provided HTML)
* - Records scraper run to history on success
* - Records scraper run to history on failure
* - Catches and logs errors from fetchPage
* - Catches and logs errors from database operations
* - Calculates newUnitsCount and rentedUnitsCount correctly
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
let mongoServer;
let client;
let db;
// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
let runScrape;
let recordScraperRun;
let createScraperIndexes;
// Store original module references so we can mock individual functions
let scraperService;
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
scraperService = require('../../services/scraperService');
({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean all relevant collections before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
// Reset all mocks
jest.restoreAllMocks();
});
// ============================================================
// Helper: Create sample HTML with units
// ============================================================
function createSampleHtml(unitCodes) {
const articles = unitCodes.map(code => `
<article
data-spaces-id="100"
data-spaces-unit="${code}"
data-spaces-unit-id="200"
data-spaces-unit-floor="5"
data-spaces-sort-area="750"
data-spaces-sort-bed="1"
data-spaces-sort-bath="1"
data-spaces-sort-price="1500"
data-spaces-available="true"
data-spaces-unavailable="false"
data-spaces-soonest="Now"
data-spaces-sort-date="1700000000"
data-spaces-plan-id="10"
data-spaces-sort-plan-name="Studio"
data-spaces-obj="unit"
data-spaces-community="TestCommunity"
data-spaces-asset="1"
data-spaces-href="/unit/${code}"
data-spaces-inventory-href="/inventory/${code}"
>
<img src="http://example.com/${code}.jpg" />
</article>
`).join('\n');
return `
<html><body>
<section class="spaces__tab-unit">
${articles}
</section>
</body></html>
`;
}
// ============================================================
// Test: recordScraperRun
// ============================================================
describe('recordScraperRun', () => {
it('should insert a run record into scraper_runs collection', async () => {
const runData = {
jobId: 'test-job-001',
trigger: 'manual',
status: 'success',
startedAt: new Date().toISOString(),
completedAt: new Date().toISOString(),
duration: 1234,
unitsProcessed: 10,
pricesInserted: 8,
errors: []
};
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
const result = await recordScraperRun(db, runData, mockLogger);
expect(result).toBeTruthy();
expect(result.insertedId).toBeTruthy();
// Verify it was inserted
const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
expect(saved).toBeTruthy();
expect(saved.status).toBe('success');
expect(saved.recordedAt).toBeInstanceOf(Date);
});
it('should not throw when insert fails', async () => {
// Pass null db to cause an error
const runData = { jobId: 'test-fail', status: 'success' };
// Should not throw
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
const result = await recordScraperRun(null, runData, mockLogger);
expect(result).toBeNull();
expect(mockLogger.error).toHaveBeenCalled();
});
});
// ============================================================
// Test: runScrape - Full workflow
// ============================================================
describe('runScrape', () => {
it('should complete full workflow with mocked dependencies', async () => {
const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
const result = await runScrape(db, {
trigger: 'manual',
jobId: 'test-full-workflow',
htmlContent: html
});
expect(result).toBeTruthy();
expect(result.jobId).toBe('test-full-workflow');
expect(result.status).toBe('success');
expect(result.trigger).toBe('manual');
expect(result.unitsProcessed).toBe(2);
expect(result.errors).toEqual([]);
});
it('should return correct result structure with jobId, status, metrics', async () => {
const html = createSampleHtml(['UNIT-X']);
const result = await runScrape(db, {
jobId: 'test-structure',
htmlContent: html
});
// Verify all expected fields exist
expect(result).toHaveProperty('jobId', 'test-structure');
expect(result).toHaveProperty('trigger', 'manual');
expect(result).toHaveProperty('dryRun', false);
expect(result).toHaveProperty('status', 'success');
expect(result).toHaveProperty('startedAt');
expect(result).toHaveProperty('completedAt');
expect(result).toHaveProperty('duration');
expect(result).toHaveProperty('unitsProcessed');
expect(result).toHaveProperty('pricesInserted');
expect(result).toHaveProperty('newUnitsCount');
expect(result).toHaveProperty('rentedUnitsCount');
expect(result).toHaveProperty('staleUnitsCount');
expect(result).toHaveProperty('errors');
// Verify types
expect(typeof result.duration).toBe('number');
expect(result.duration).toBeGreaterThanOrEqual(0);
expect(typeof result.startedAt).toBe('string');
expect(typeof result.completedAt).toBe('string');
expect(Array.isArray(result.errors)).toBe(true);
});
it('should generate a jobId when not provided', async () => {
const html = createSampleHtml(['UNIT-GEN']);
const result = await runScrape(db, {
htmlContent: html
});
expect(result.jobId).toBeTruthy();
expect(typeof result.jobId).toBe('string');
// UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
});
// ============================================================
// Test: dryRun option
// ============================================================
it('should skip DB writes when dryRun is true', async () => {
const html = createSampleHtml(['DRY-A', 'DRY-B']);
const result = await runScrape(db, {
jobId: 'test-dry-run',
htmlContent: html,
dryRun: true
});
expect(result.status).toBe('success');
expect(result.dryRun).toBe(true);
expect(result.unitsProcessed).toBe(2);
expect(result.pricesInserted).toBe(0);
expect(result.staleUnitsCount).toBe(0);
// Verify no units were written to the units collection
const unitsCount = await db.collection('units_scraper').countDocuments();
expect(unitsCount).toBe(0);
// Verify no prices were written
const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
expect(pricesCount).toBe(0);
});
// ============================================================
// Test: htmlContent option
// ============================================================
it('should use provided HTML instead of fetching when htmlContent is given', async () => {
const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
// Mock axios.get to track if HTTP fetch is attempted
const axios = require('axios');
const axiosSpy = jest.spyOn(axios, 'get');
const result = await runScrape(db, {
jobId: 'test-html-content',
htmlContent: html
});
expect(result.status).toBe('success');
expect(result.unitsProcessed).toBe(3);
// axios.get should NOT have been called since we provided htmlContent
expect(axiosSpy).not.toHaveBeenCalled();
});
// ============================================================
// Test: Records scraper run on success
// ============================================================
it('should record scraper run to history on success', async () => {
const html = createSampleHtml(['REC-A']);
await runScrape(db, {
jobId: 'test-record-success',
htmlContent: html
});
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('success');
expect(runRecord.recordedAt).toBeInstanceOf(Date);
expect(runRecord.unitsProcessed).toBe(1);
});
// ============================================================
// Test: Records scraper run on failure
// ============================================================
it('should record scraper run to history on failure', async () => {
const html = createSampleHtml(['FAIL-A']);
// Create a db proxy that throws on bulkWrite (used by upsertUnits)
// but allows other operations (like scraper_runs insertOne) to pass through
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'units_scraper') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
return async () => {
throw new Error('Database connection lost');
};
}
const value = target[prop];
if (typeof value === 'function') {
return value.bind(target);
}
return value;
}
});
}
return realCollection;
}
};
const result = await runScrape(faultyDb, {
jobId: 'test-record-failure',
htmlContent: html
});
expect(result.status).toBe('failed');
expect(result.errors).toContain('Database connection lost');
// The run should still be recorded (via the real db passed through for scraper_runs)
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('failed');
expect(runRecord.errors).toContain('Database connection lost');
});
// ============================================================
// Test: Catches errors from fetchPage
// ============================================================
it('should catch and handle errors from fetchPage', async () => {
// Mock axios.get to throw a network error (fetchPage uses axios internally)
const axios = require('axios');
jest.spyOn(axios, 'get').mockRejectedValue(
new Error('Network timeout')
);
const result = await runScrape(db, {
jobId: 'test-fetch-error'
// No htmlContent, so it will call fetchPage which uses axios
});
expect(result.status).toBe('failed');
expect(result.errors).toContain('Network timeout');
expect(result.unitsProcessed).toBe(0);
});
// ============================================================
// Test: Catches errors from database operations
// ============================================================
it('should catch and handle errors from database operations', async () => {
const html = createSampleHtml(['DB-ERR']);
// Create a db proxy that throws on the prices collection bulkWrite
// Use Proxy to properly delegate all methods to the real collection
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'unit_prices_scraper') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
return async () => {
throw new Error('Write concern timeout');
};
}
const value = target[prop];
if (typeof value === 'function') {
return value.bind(target);
}
return value;
}
});
}
return realCollection;
}
};
const result = await runScrape(faultyDb, {
jobId: 'test-db-error',
htmlContent: html
});
expect(result.status).toBe('failed');
expect(result.errors).toContain('Write concern timeout');
// Run should still be recorded despite error
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('failed');
});
// ============================================================
// Test: Calculates newUnitsCount and rentedUnitsCount correctly
// ============================================================
it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
// First, simulate yesterday's data by inserting price records for yesterday
const today = new Date();
const yesterday = new Date(today);
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
const yesterdayStr = yesterday.toISOString().split('T')[0];
// Yesterday had units: OLD-A, OLD-B, OLD-C
await db.collection('unit_prices_scraper').insertMany([
{ unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
{ unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
{ unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
]);
// Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
const result = await runScrape(db, {
jobId: 'test-calc-changes',
htmlContent: html
});
expect(result.status).toBe('success');
// NEW-D is new (not in yesterday's data)
expect(result.newUnitsCount).toBe(1);
// OLD-C was in yesterday's data but not in today's scrape
expect(result.rentedUnitsCount).toBe(1);
expect(result.unitsProcessed).toBe(3);
});
// ============================================================
// Test: Default trigger is 'manual'
// ============================================================
it('should default trigger to manual', async () => {
const html = createSampleHtml(['DEF-A']);
const result = await runScrape(db, {
jobId: 'test-default-trigger',
htmlContent: html
});
expect(result.trigger).toBe('manual');
});
// ============================================================
// Test: Supports scheduled trigger
// ============================================================
it('should support scheduled trigger', async () => {
const html = createSampleHtml(['SCHED-A']);
const result = await runScrape(db, {
jobId: 'test-scheduled-trigger',
trigger: 'scheduled',
htmlContent: html
});
expect(result.trigger).toBe('scheduled');
});
// ============================================================
// Test: Handles empty HTML (no units found)
// ============================================================
it('should handle HTML with no units gracefully', async () => {
const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
const result = await runScrape(db, {
jobId: 'test-empty-html',
htmlContent: emptyHtml
});
expect(result.status).toBe('success');
expect(result.unitsProcessed).toBe(0);
expect(result.errors).toContain('No units found in HTML');
});
});
// ============================================================
// Test: sanitizeError - Error message sanitization
// ============================================================
describe('sanitizeError', () => {
let sanitizeError;
beforeAll(() => {
({ sanitizeError } = require('../../services/scraperService'));
});
// ----------------------------------------------------------
// File path sanitization
// ----------------------------------------------------------
describe('file path sanitization', () => {
it('should remove Unix absolute file paths from error messages', () => {
const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/home\/user/);
expect(sanitized.message).toContain('ENOENT');
});
it('should remove /var paths from error messages', () => {
const error = new Error('Failed to read /var/app/current/data/secrets.yml');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/var\/app/);
expect(sanitized.message).toContain('Failed to read');
});
it('should remove Windows-style file paths from error messages', () => {
const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/C:\\Users/);
expect(sanitized.message).toContain('Cannot find module');
});
it('should remove /tmp and /usr paths from error messages', () => {
const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/\/tmp\//);
expect(sanitized.message).not.toMatch(/\/usr\//);
});
});
// ----------------------------------------------------------
// MongoDB connection string sanitization
// ----------------------------------------------------------
describe('connection string sanitization', () => {
it('should redact mongodb:// connection strings', () => {
const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('s3cretP4ss');
expect(sanitized.message).not.toContain('admin:');
expect(sanitized.message).toContain('Connection failed');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact mongodb+srv:// connection strings', () => {
const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('password123');
expect(sanitized.message).not.toContain('user:');
expect(sanitized.message).toContain('Timeout connecting to');
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
it('should redact connection string without credentials', () => {
const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
});
});
// ----------------------------------------------------------
// Credential / secret sanitization
// ----------------------------------------------------------
describe('credential sanitization', () => {
it('should redact common environment variable patterns', () => {
const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('sk-abc123xyz');
expect(sanitized.message).not.toContain('bearer-9876');
});
it('should redact password patterns', () => {
const error = new Error('Auth failed with password=MyS3cret!');
const sanitized = sanitizeError(error);
expect(sanitized.message).not.toContain('MyS3cret!');
});
});
// ----------------------------------------------------------
// Error type preservation
// ----------------------------------------------------------
describe('error type preservation', () => {
it('should preserve the error type/name', () => {
const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TypeError');
});
it('should preserve custom error names', () => {
const error = new Error('Timeout at /var/app/scraper.js:100');
error.name = 'TimeoutError';
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('TimeoutError');
});
it('should preserve error name for RangeError', () => {
const error = new RangeError('Maximum call stack size exceeded');
const sanitized = sanitizeError(error);
expect(sanitized.name).toBe('RangeError');
});
});
// ----------------------------------------------------------
// General description preserved for debugging
// ----------------------------------------------------------
describe('general description preservation', () => {
it('should preserve a useful general description', () => {
const error = new Error('Network timeout after 30000ms');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('Network timeout after 30000ms');
});
it('should preserve error description when no sensitive data present', () => {
const error = new Error('Request failed with status code 500');
const sanitized = sanitizeError(error);
expect(sanitized.message).toBe('Request failed with status code 500');
});
it('should return a useful message even after heavy sanitization', () => {
const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
const sanitized = sanitizeError(error);
expect(sanitized.message).toContain('ECONNREFUSED');
expect(sanitized.message.length).toBeGreaterThan(5);
});
});
// ----------------------------------------------------------
// Stack trace sanitization
// ----------------------------------------------------------
describe('stack trace removal', () => {
it('should remove file paths from stack traces', () => {
const error = new Error('Something failed');
error.stack = 'Error: Something failed\n at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
const sanitized = sanitizeError(error);
expect(sanitized.stack).not.toMatch(/\/home\/user/);
expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
});
it('should handle errors without stack trace', () => {
const error = new Error('No stack');
error.stack = undefined;
const sanitized = sanitizeError(error);
expect(sanitized.stack).toBeUndefined();
});
});
// ----------------------------------------------------------
// Integration: sanitization applied before recordScraperRun()
// ----------------------------------------------------------
describe('sanitization before recordScraperRun()', () => {
it('should store sanitized error message in scraper_runs on failure', async () => {
const html = createSampleHtml(['SANITIZE-A']);
// Create a db proxy that throws an error containing sensitive info
const sensitiveError = new Error(
'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
);
sensitiveError.name = 'MongoServerError';
const faultyDb = {
collection: (name) => {
const realCollection = db.collection(name);
if (name === 'units_scraper') {
return new Proxy(realCollection, {
get(target, prop) {
if (prop === 'bulkWrite') {
return async () => { throw sensitiveError; };
}
const value = target[prop];
if (typeof value === 'function') {
return value.bind(target);
}
return value;
}
});
}
return realCollection;
}
};
const result = await runScrape(faultyDb, {
jobId: 'test-sanitized-error',
htmlContent: html
});
expect(result.status).toBe('failed');
// Verify the error stored in result.errors is sanitized
const errorMsg = result.errors[0];
expect(errorMsg).not.toContain('SuperSecret');
expect(errorMsg).not.toContain('admin:');
expect(errorMsg).not.toContain('/home/deploy/');
expect(errorMsg).toContain('MongoServerError');
// Verify the error stored in scraper_runs is sanitized
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
expect(runRecord).toBeTruthy();
expect(runRecord.status).toBe('failed');
const storedError = runRecord.errors[0];
expect(storedError).not.toContain('SuperSecret');
expect(storedError).not.toContain('admin:');
expect(storedError).not.toContain('/home/deploy/');
});
});
});
// ============================================================
// Test: createScraperIndexes
// ============================================================
describe('createScraperIndexes', () => {
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
beforeEach(() => {
mockLogger.info.mockClear();
mockLogger.warn.mockClear();
mockLogger.error.mockClear();
});
it('should create the status_startedAt compound index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
expect(statusIndex).toBeTruthy();
expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
});
it('should create the startedAt_desc index', async () => {
await createScraperIndexes(db, mockLogger);
const indexes = await db.collection('scraper_runs').indexes();
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(startedAtIndex).toBeTruthy();
expect(startedAtIndex.key).toEqual({ startedAt: -1 });
});
it('should log success after creating indexes', async () => {
await createScraperIndexes(db, mockLogger);
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
});
it('should be idempotent (no error on second call)', async () => {
await createScraperIndexes(db, mockLogger);
// Calling again should not throw
await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
// Indexes should still exist
const indexes = await db.collection('scraper_runs').indexes();
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
expect(statusIndex).toBeTruthy();
expect(startedAtIndex).toBeTruthy();
});
it('should handle database errors gracefully', async () => {
// Pass an object that will throw when collection() is called
const faultyDb = {
collection: () => {
throw new Error('Connection lost');
}
};
await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
});
});

177
__tests__/scraper/serverIntegration.test.js
View File

@ -1,177 +0,0 @@
/**
* Tests for scraper integration in server.js startup
*
* Covers:
* - Server initializes scraper scheduler on startup
* - Server does not crash if scheduler init fails
* - Indexes are created before scheduler starts
* - Scheduler not initialized when SCRAPER_ENABLED=false
*
* Strategy: Instead of importing server.js directly (which starts Express and
* connects to MongoDB), we replicate the scraper initialization logic from
* connectToMongoDB() and test it with mocked dependencies. This verifies the
* integration contract without needing a full server boot.
*/
const { MongoClient } = require('mongodb');
describe('Server Scraper Integration', () => {
let connection;
let db;
beforeAll(async () => {
const uri = process.env.MONGO_URI;
connection = await MongoClient.connect(uri);
db = connection.db('apartments_server_integration_test');
});
afterAll(async () => {
if (connection) {
await connection.close();
}
});
describe('scraper initialization during startup', () => {
let mockCreateScraperIndexes;
let mockInitializeScheduler;
let mockCreateLogger;
let mockLogger;
beforeEach(() => {
mockLogger = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
debug: jest.fn()
};
mockCreateLogger = jest.fn(() => mockLogger);
mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
mockInitializeScheduler = jest.fn();
});
/**
* Replicates the scraper initialization logic from server.js connectToMongoDB().
* This is the exact pattern used in production:
*
* try {
* const scraperLogger = createLogger('server-init');
* await createScraperIndexes(db, scraperLogger);
* if (scraperConfig.SCRAPER_ENABLED) {
* initializeScheduler(db);
* }
* } catch (error) {
* console.error('Scraper initialization failed (server continues):', error.message);
* }
*/
async function runScraperInit(testDb, config) {
try {
const scraperLogger = mockCreateLogger('server-init');
await mockCreateScraperIndexes(testDb, scraperLogger);
if (config.SCRAPER_ENABLED) {
mockInitializeScheduler(testDb);
}
return { success: true };
} catch (error) {
return { success: false, error: error.message };
}
}
it('should initialize scraper scheduler on startup when enabled', async () => {
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
});
it('should not crash if createScraperIndexes throws an error', async () => {
mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
const config = { SCRAPER_ENABLED: true };
const result = await runScraperInit(db, config);
// The server should continue (error is caught, not thrown)
expect(result.success).toBe(false);
expect(result.error).toBe('Index creation failed');
// initializeScheduler should NOT have been called since the error occurred before it
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
it('should create indexes before initializing the scheduler', async () => {
const callOrder = [];
mockCreateScraperIndexes.mockImplementation(async () => {
callOrder.push('createScraperIndexes');
});
mockInitializeScheduler.mockImplementation(() => {
callOrder.push('initializeScheduler');
});
const config = { SCRAPER_ENABLED: true };
await runScraperInit(db, config);
expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
});
it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
const config = { SCRAPER_ENABLED: false };
const result = await runScraperInit(db, config);
expect(result.success).toBe(true);
expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
expect(mockInitializeScheduler).not.toHaveBeenCalled();
});
});
describe('server.js imports and exports verification', () => {
it('should be able to import createScraperIndexes from scraperService', () => {
const { createScraperIndexes } = require('../../services/scraperService');
expect(typeof createScraperIndexes).toBe('function');
});
it('should be able to import initializeScheduler from scraperJob', () => {
const { initializeScheduler } = require('../../jobs/scraperJob');
expect(typeof initializeScheduler).toBe('function');
});
it('should be able to import registerSignalHandlers from scraperJob', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
expect(typeof registerSignalHandlers).toBe('function');
});
it('should be able to import SCRAPER_ENABLED from scraper config', () => {
const scraperConfig = require('../../config/scraper');
expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
});
});
describe('createScraperIndexes error handling with real db', () => {
it('should handle index creation gracefully with a valid db', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const { createLogger } = require('../../services/scraperLogger');
const logger = createLogger('test-server-init');
// Should not throw - creates indexes on the test db
await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
});
it('should throw when given an invalid db object', async () => {
const { createScraperIndexes } = require('../../services/scraperService');
const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
// Passing null should throw (simulates what the try/catch in server.js protects against)
await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
});
});
describe('registerSignalHandlers', () => {
it('should register SIGTERM and SIGINT handlers without throwing', () => {
const { registerSignalHandlers } = require('../../jobs/scraperJob');
// Should not throw when called
expect(() => registerSignalHandlers()).not.toThrow();
});
});
});

561
__tests__/scraper/updateDailySummary.test.js
View File

@ -1,561 +0,0 @@
/**
* Tests for updateDailySummary() daily summary aggregation
*
* Covers:
* - Upsert by date key (YYYY-MM-DD)
* - Summary includes new_units, rented_units, stale_units arrays
* - Count fields are calculated correctly
* - net_change = newUnits.length - rentedUnits.length
* - total_available_today is set correctly
* - total_available_yesterday fetched from previous day
* - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
* - turnover_rate is 0 when yesterdayTotal is 0
* - Error handling when updateOne fails
* - Logging of summary update details
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
// Will require updateDailySummary after implementation
let updateDailySummary;
let mongoServer;
let client;
let db;
const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
// Mock logger for capturing log calls
function createMockLogger() {
return {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
}
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
({ updateDailySummary } = require('../../services/scraperService'));
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean collections before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
});
describe('updateDailySummary', () => {
// ---------------------------------------------------------------
// 1. Upsert by date key
// ---------------------------------------------------------------
describe('upsert by date key', () => {
it('should insert a new summary when no document exists for the date', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102'],
rentedUnits: ['B201'],
staleUnitsCount: 2,
totalAvailable: 15
};
const result = await updateDailySummary(db, summaryData, logger);
// Should have upserted (created new)
expect(result.upsertedCount).toBe(1);
// Verify the document in the database
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc).not.toBeNull();
expect(doc.date).toBe('2026-02-05');
});
it('should update an existing summary for the same date (idempotent upsert)', async () => {
const logger = createMockLogger();
// Insert initial summary
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-02-05',
new_units: ['A101'],
rented_units: [],
total_available_today: 10
});
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102', 'A103'],
rentedUnits: ['B201'],
staleUnitsCount: 1,
totalAvailable: 12
};
const result = await updateDailySummary(db, summaryData, logger);
// Should have matched and modified (not upserted)
expect(result.matchedCount).toBe(1);
// Verify updated values
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
expect(doc.total_available_today).toBe(12);
// Verify only one document exists for this date
const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
expect(count).toBe(1);
});
});
// ---------------------------------------------------------------
// 2. Summary includes new_units, rented_units, stale_units arrays
// ---------------------------------------------------------------
describe('summary arrays', () => {
it('should store new_units, rented_units, and stale_units arrays', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102'],
rentedUnits: ['B201', 'B202', 'B203'],
staleUnitsCount: 3,
totalAvailable: 20
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.new_units).toEqual(['A101', 'A102']);
expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
expect(doc.stale_units).toEqual([]);
});
it('should handle empty arrays for new_units and rented_units', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 25
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.new_units).toEqual([]);
expect(doc.rented_units).toEqual([]);
expect(doc.stale_units).toEqual([]);
});
});
// ---------------------------------------------------------------
// 3. Count fields are calculated correctly
// ---------------------------------------------------------------
describe('count fields', () => {
it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102', 'A103'],
rentedUnits: ['B201'],
staleUnitsCount: 5,
totalAvailable: 30
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.new_units_count).toBe(3);
expect(doc.rented_units_count).toBe(1);
expect(doc.stale_units_count).toBe(5);
});
it('should set all count fields to 0 when arrays are empty', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 10
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.new_units_count).toBe(0);
expect(doc.rented_units_count).toBe(0);
expect(doc.stale_units_count).toBe(0);
});
});
// ---------------------------------------------------------------
// 4. net_change = newUnits.length - rentedUnits.length
// ---------------------------------------------------------------
describe('net_change calculation', () => {
it('should calculate positive net_change when more units added than rented', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102', 'A103'],
rentedUnits: ['B201'],
staleUnitsCount: 0,
totalAvailable: 20
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.net_change).toBe(2); // 3 - 1 = 2
});
it('should calculate negative net_change when more units rented than added', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: ['B201', 'B202', 'B203', 'B204'],
staleUnitsCount: 0,
totalAvailable: 10
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.net_change).toBe(-3); // 1 - 4 = -3
});
it('should calculate zero net_change when equal new and rented', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102'],
rentedUnits: ['B201', 'B202'],
staleUnitsCount: 0,
totalAvailable: 15
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.net_change).toBe(0);
});
});
// ---------------------------------------------------------------
// 5. total_available_today is set correctly
// ---------------------------------------------------------------
describe('total_available_today', () => {
it('should store the totalAvailable value as total_available_today', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 42
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.total_available_today).toBe(42);
});
it('should handle totalAvailable of 0', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: ['B201'],
staleUnitsCount: 0,
totalAvailable: 0
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.total_available_today).toBe(0);
});
});
// ---------------------------------------------------------------
// 6. total_available_yesterday fetched from previous day
// ---------------------------------------------------------------
describe('total_available_yesterday', () => {
it('should fetch total_available_today from previous day summary', async () => {
const logger = createMockLogger();
// Insert yesterday's summary
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-02-04',
total_available_today: 35,
new_units: [],
rented_units: []
});
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: ['B201'],
staleUnitsCount: 0,
totalAvailable: 35
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.total_available_yesterday).toBe(35);
});
it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 20
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.total_available_yesterday).toBe(0);
});
it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
const logger = createMockLogger();
// Insert March 31 summary
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-03-31',
total_available_today: 50
});
const summaryData = {
date: '2026-04-01',
newUnits: [],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 48
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
expect(doc.total_available_yesterday).toBe(50);
});
});
// ---------------------------------------------------------------
// 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
// ---------------------------------------------------------------
describe('turnover_rate calculation', () => {
it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
const logger = createMockLogger();
// Insert yesterday's summary with 50 total available
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-02-04',
total_available_today: 50
});
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
staleUnitsCount: 0,
totalAvailable: 46
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
// 5 rented / 50 yesterday = 10%
expect(doc.turnover_rate).toBe(10);
});
it('should round turnover_rate to 2 decimal places', async () => {
const logger = createMockLogger();
// Insert yesterday's summary with 30 total available
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-02-04',
total_available_today: 30
});
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: ['B201', 'B202'],
staleUnitsCount: 0,
totalAvailable: 28
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
// 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
expect(doc.turnover_rate).toBe(6.67);
});
});
// ---------------------------------------------------------------
// 8. turnover_rate is 0 when yesterdayTotal is 0
// ---------------------------------------------------------------
describe('turnover_rate edge cases', () => {
it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: ['B201'],
staleUnitsCount: 0,
totalAvailable: 15
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.turnover_rate).toBe(0);
});
it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
const logger = createMockLogger();
// Insert yesterday's summary with 0 total available
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
date: '2026-02-04',
total_available_today: 0
});
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: ['B201'],
staleUnitsCount: 0,
totalAvailable: 10
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.turnover_rate).toBe(0);
});
it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 10
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.turnover_rate).toBe(0);
});
});
// ---------------------------------------------------------------
// 9. Error handling when updateOne fails
// ---------------------------------------------------------------
describe('error handling', () => {
it('should throw and log error when updateOne fails', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101'],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 10
};
// Create a mock db that throws on updateOne
const mockCollection = {
findOne: jest.fn().mockResolvedValue(null),
updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await expect(
updateDailySummary(mockDb, summaryData, logger)
).rejects.toThrow('Write concern timeout');
// Verify error was logged
expect(logger.error).toHaveBeenCalledWith(
'Failed to update daily summary',
expect.objectContaining({
errorType: 'Error',
errorMessage: 'Write concern timeout'
})
);
});
});
// ---------------------------------------------------------------
// 10. Logging of summary update details
// ---------------------------------------------------------------
describe('logging', () => {
it('should log summary update details on success', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: ['A101', 'A102'],
rentedUnits: ['B201'],
staleUnitsCount: 1,
totalAvailable: 18
};
await updateDailySummary(db, summaryData, logger);
expect(logger.info).toHaveBeenCalledWith(
'Daily summary updated',
expect.objectContaining({
date: '2026-02-05',
newUnits: 2,
rentedUnits: 1,
totalAvailable: 18
})
);
});
it('should include timestamp field in the stored summary', async () => {
const logger = createMockLogger();
const summaryData = {
date: '2026-02-05',
newUnits: [],
rentedUnits: [],
staleUnitsCount: 0,
totalAvailable: 10
};
await updateDailySummary(db, summaryData, logger);
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
expect(doc.timestamp).toBeDefined();
expect(typeof doc.timestamp).toBe('string');
// Verify it is a valid ISO timestamp
expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
});
});
});

498
__tests__/scraper/upsertUnits.test.js
View File

@ -1,498 +0,0 @@
/**
* Tests for upsertUnits() bulk operation
*
* Covers:
* - bulkWrite is called with correct updateOne operations
* - upsert: true is set for all operations
* - $set updates last_scraped and data_source fields
* - $setOnInsert sets first_seen for new units
* - Empty units array logs warning and returns early
* - ordered: false is set for parallel execution
* - Error handling when bulkWrite fails
* - Logging of matched, modified, and upserted counts
*/
const { MongoClient } = require('mongodb');
const { MongoMemoryServer } = require('mongodb-memory-server');
// We will require upsertUnits after implementation
let upsertUnits;
let mongoServer;
let client;
let db;
// Mock logger for capturing log calls
function createMockLogger() {
return {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn()
};
}
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
const uri = mongoServer.getUri();
client = new MongoClient(uri);
await client.connect();
db = client.db('test_apartments');
// Dynamically require to pick up implementation
({ upsertUnits } = require('../../services/scraperService'));
});
afterAll(async () => {
if (client) await client.close();
if (mongoServer) await mongoServer.stop();
});
beforeEach(async () => {
// Clean the units collection before each test
const collections = await db.listCollections().toArray();
for (const col of collections) {
await db.collection(col.name).deleteMany({});
}
});
describe('upsertUnits', () => {
const UNITS_COLLECTION = 'units_scraper';
// ---------------------------------------------------------------
// 1. bulkWrite is called with correct updateOne operations
// ---------------------------------------------------------------
describe('bulkWrite operations', () => {
it('should create updateOne operations for each unit', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
{ unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
];
const result = await upsertUnits(db, units, logger);
// Both units should be upserted (new inserts)
expect(result.upsertedCount).toBe(2);
// Verify documents exist in the collection
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(2);
const unitCodes = docs.map(d => d.unit_code).sort();
expect(unitCodes).toEqual(['A101', 'B202']);
});
it('should filter by unit_code as the unique key', async () => {
const logger = createMockLogger();
// Insert a unit first
const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
await upsertUnits(db, units, logger);
// Upsert again with updated price
const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
const result = await upsertUnits(db, updatedUnits, logger);
// Should match and modify, not insert new
expect(result.matchedCount).toBe(1);
expect(result.upsertedCount).toBe(0);
// Verify only one document exists
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
expect(docs).toHaveLength(1);
expect(docs[0].price).toBe(1400);
});
});
// ---------------------------------------------------------------
// 2. upsert: true is set for all operations
// ---------------------------------------------------------------
describe('upsert behavior', () => {
it('should insert new units when they do not exist (upsert: true)', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'NEW-001', price: 1100 },
{ unit_code: 'NEW-002', price: 1200 },
{ unit_code: 'NEW-003', price: 1300 }
];
const result = await upsertUnits(db, units, logger);
expect(result.upsertedCount).toBe(3);
// Verify all three are in the database
const count = await db.collection(UNITS_COLLECTION).countDocuments();
expect(count).toBe(3);
});
it('should update existing units without creating duplicates', async () => {
const logger = createMockLogger();
// First insert
await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
// Second upsert with same unit_code
await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
const count = await db.collection(UNITS_COLLECTION).countDocuments();
expect(count).toBe(1);
});
});
// ---------------------------------------------------------------
// 3. $set updates last_scraped and data_source fields
// ---------------------------------------------------------------
describe('$set fields', () => {
it('should set last_scraped timestamp on every upsert', async () => {
const logger = createMockLogger();
const beforeTime = new Date().toISOString();
const units = [{ unit_code: 'T100', price: 1000 }];
await upsertUnits(db, units, logger);
const afterTime = new Date().toISOString();
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
expect(doc.last_scraped).toBeDefined();
expect(typeof doc.last_scraped).toBe('string');
// The last_scraped should be between before and after timestamps
expect(doc.last_scraped >= beforeTime).toBe(true);
expect(doc.last_scraped <= afterTime).toBe(true);
});
it('should set data_source to "web_scraper"', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'T200', price: 1100 }];
await upsertUnits(db, units, logger);
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
expect(doc.data_source).toBe('web_scraper');
});
it('should update last_scraped on subsequent upserts', async () => {
const logger = createMockLogger();
// First insert
await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
const firstScraped = doc1.last_scraped;
// Small delay to ensure different timestamp
await new Promise(resolve => setTimeout(resolve, 10));
// Second upsert
await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
const secondScraped = doc2.last_scraped;
expect(secondScraped > firstScraped).toBe(true);
});
it('should spread all unit fields into the $set operation', async () => {
const logger = createMockLogger();
const units = [{
unit_code: 'T400',
price: 1500,
bed_count: 2,
bath_count: 1.5,
area: 900,
floor: 3,
plan_name: 'Luxury',
community: 'Tower A'
}];
await upsertUnits(db, units, logger);
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
expect(doc.price).toBe(1500);
expect(doc.bed_count).toBe(2);
expect(doc.bath_count).toBe(1.5);
expect(doc.area).toBe(900);
expect(doc.floor).toBe(3);
expect(doc.plan_name).toBe('Luxury');
expect(doc.community).toBe('Tower A');
});
});
// ---------------------------------------------------------------
// 4. $setOnInsert sets first_seen for new units
// ---------------------------------------------------------------
describe('$setOnInsert for first_seen', () => {
it('should set first_seen on initial insert', async () => {
const logger = createMockLogger();
const beforeTime = new Date().toISOString();
await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
expect(doc.first_seen).toBeDefined();
expect(typeof doc.first_seen).toBe('string');
expect(doc.first_seen >= beforeTime).toBe(true);
});
it('should NOT update first_seen on subsequent upserts', async () => {
const logger = createMockLogger();
// First insert
await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
const originalFirstSeen = doc1.first_seen;
// Small delay to ensure different timestamp
await new Promise(resolve => setTimeout(resolve, 10));
// Second upsert (update)
await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
// first_seen should remain unchanged
expect(doc2.first_seen).toBe(originalFirstSeen);
});
it('should set different first_seen for different units inserted at different times', async () => {
const logger = createMockLogger();
await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
await new Promise(resolve => setTimeout(resolve, 10));
await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
// Both should have first_seen, but different values
expect(doc1.first_seen).toBeDefined();
expect(doc2.first_seen).toBeDefined();
expect(doc2.first_seen > doc1.first_seen).toBe(true);
});
});
// ---------------------------------------------------------------
// 5. Empty units array logs warning and returns early
// ---------------------------------------------------------------
describe('empty units array', () => {
it('should log a warning when units array is empty', async () => {
const logger = createMockLogger();
await upsertUnits(db, [], logger);
expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
});
it('should return early with zero counts for empty array', async () => {
const logger = createMockLogger();
const result = await upsertUnits(db, [], logger);
expect(result.modifiedCount).toBe(0);
expect(result.upsertedCount).toBe(0);
});
it('should not perform any database writes for empty array', async () => {
const logger = createMockLogger();
await upsertUnits(db, [], logger);
const count = await db.collection(UNITS_COLLECTION).countDocuments();
expect(count).toBe(0);
});
});
// ---------------------------------------------------------------
// 6. ordered: false is set for parallel execution
// ---------------------------------------------------------------
describe('ordered: false for parallel execution', () => {
it('should successfully process all units even if one has a conflict', async () => {
const logger = createMockLogger();
// Insert multiple units - ordered: false means all operations execute
// even if some fail. We verify by checking all valid units are present.
const units = [
{ unit_code: 'P100', price: 1000 },
{ unit_code: 'P200', price: 1100 },
{ unit_code: 'P300', price: 1200 },
{ unit_code: 'P400', price: 1300 },
{ unit_code: 'P500', price: 1400 }
];
const result = await upsertUnits(db, units, logger);
expect(result.upsertedCount).toBe(5);
// All 5 units should be in the database
const count = await db.collection(UNITS_COLLECTION).countDocuments();
expect(count).toBe(5);
});
it('should handle a mix of new inserts and existing updates', async () => {
const logger = createMockLogger();
// First, insert some units
await upsertUnits(db, [
{ unit_code: 'M100', price: 900 },
{ unit_code: 'M200', price: 1000 }
], logger);
// Now upsert a mix of existing and new units
const mixedUnits = [
{ unit_code: 'M100', price: 950 }, // existing - update
{ unit_code: 'M200', price: 1050 }, // existing - update
{ unit_code: 'M300', price: 1100 }, // new - insert
];
const result = await upsertUnits(db, mixedUnits, logger);
expect(result.matchedCount).toBe(2);
expect(result.upsertedCount).toBe(1);
// Total should be 3 documents
const count = await db.collection(UNITS_COLLECTION).countDocuments();
expect(count).toBe(3);
});
});
// ---------------------------------------------------------------
// 7. Error handling when bulkWrite fails
// ---------------------------------------------------------------
describe('error handling', () => {
it('should throw error when bulkWrite fails', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'E100', price: 1000 }];
// Create a mock db that throws on bulkWrite
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
});
it('should log error details when bulkWrite fails', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'E200', price: 1000 }];
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
try {
await upsertUnits(mockDb, units, logger);
} catch (e) {
// Expected to throw
}
expect(logger.error).toHaveBeenCalledWith(
'Failed to upsert units',
expect.objectContaining({
errorType: 'Error',
errorMessage: 'Write concern timeout'
})
);
});
it('should re-throw the original error', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'E300', price: 1000 }];
const originalError = new TypeError('Invalid operation');
const mockCollection = {
bulkWrite: jest.fn().mockRejectedValue(originalError)
};
const mockDb = {
collection: jest.fn().mockReturnValue(mockCollection)
};
await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
});
});
// ---------------------------------------------------------------
// 8. Logging of matched, modified, and upserted counts
// ---------------------------------------------------------------
describe('logging of result counts', () => {
it('should log matched, modified, and upserted counts on success', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'L100', price: 1000 },
{ unit_code: 'L200', price: 1100 }
];
await upsertUnits(db, units, logger);
expect(logger.info).toHaveBeenCalledWith(
'Units upserted',
expect.objectContaining({
matched: expect.any(Number),
modified: expect.any(Number),
upserted: expect.any(Number)
})
);
});
it('should log correct counts for new inserts', async () => {
const logger = createMockLogger();
const units = [
{ unit_code: 'L300', price: 1000 },
{ unit_code: 'L400', price: 1100 }
];
await upsertUnits(db, units, logger);
// For new inserts: matched=0, modified=0, upserted=2
expect(logger.info).toHaveBeenCalledWith(
'Units upserted',
expect.objectContaining({
matched: 0,
modified: 0,
upserted: 2
})
);
});
it('should log correct counts for updates to existing units', async () => {
const logger = createMockLogger();
// First insert
await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
// Reset mock to capture only the second call
logger.info.mockClear();
// Update existing
await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
expect(logger.info).toHaveBeenCalledWith(
'Units upserted',
expect.objectContaining({
matched: 1,
modified: 1,
upserted: 0
})
);
});
});
// ---------------------------------------------------------------
// 9. Return value
// ---------------------------------------------------------------
describe('return value', () => {
it('should return the bulkWrite result object', async () => {
const logger = createMockLogger();
const units = [{ unit_code: 'R100', price: 1000 }];
const result = await upsertUnits(db, units, logger);
// bulkWrite result should have these standard properties
expect(result).toHaveProperty('matchedCount');
expect(result).toHaveProperty('modifiedCount');
expect(result).toHaveProperty('upsertedCount');
});
});
});

41
config/scraper.js
View File

@ -1,41 +0,0 @@
/**
* Scraper Configuration Module
*
* Contains all constants and settings for the apartment price scraper.
* All values can be overridden via environment variables for deployment flexibility.
*
* See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
*/
module.exports = {
// Target URL for apartment listings
TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
// Scheduling configuration
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
// Retry configuration for HTTP requests
RETRY_CONFIG: {
maxRetries: 3,
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
},
// Graceful shutdown timeout (how long to wait for running job before force-stopping)
SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
// MongoDB collection names - defaults to validation collections for scraper validation;
// switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
COLLECTIONS: {
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper',
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper',
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
}
};

34
eslint.config.mjs
View File

@ -1,34 +0,0 @@
import js from '@eslint/js';
import globals from 'globals';
export default [
js.configs.recommended,
{
languageOptions: {
ecmaVersion: 2022,
sourceType: 'commonjs',
globals: {
...globals.node,
...globals.jest,
},
},
rules: {
// Allow unused vars with underscore prefix, and unused function args
'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
'no-console': 'off',
// Formatting rules as warnings (won't fail CI, but will be reported)
'semi': ['warn', 'always'],
'no-multiple-empty-lines': ['warn', { max: 2 }],
'eol-last': ['warn', 'always'],
'no-trailing-spaces': 'warn',
},
},
{
ignores: [
'node_modules/**',
'coverage/**',
'*.min.js',
'eslint.config.mjs',
],
},
];

275
jobs/scraperJob.js
View File

@ -1,275 +0,0 @@
const cron = require('node-cron');
const crypto = require('crypto');
const config = require('../config/scraper');
const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger');
// In-process mutex state
let isRunning = false;
let currentJobId = null;
// Scheduler state
let scheduledJob = null;
// Shutdown state
let shuttingDown = false;
let runningJobPromise = null;
/**
* Check if scraper is currently running
* @returns {boolean}
*/
function isScraperRunning() {
return isRunning;
}
/**
* Get current job ID if running
* @returns {string|null}
*/
function getCurrentJobId() {
return currentJobId;
}
/**
* Acquire the scraper lock
* @param {string} jobId - Job ID to set
* @returns {boolean} True if lock acquired
*/
function acquireLock(jobId) {
if (isRunning || shuttingDown) {
return false;
}
isRunning = true;
currentJobId = jobId;
return true;
}
/**
* Release the scraper lock
*/
function releaseLock() {
isRunning = false;
currentJobId = null;
}
/**
* Get the configured schedule expression
* @returns {string} Cron expression or 'disabled'
*/
function getScheduleExpression() {
if (!config.SCRAPER_ENABLED) {
return 'disabled';
}
return config.SCRAPER_SCHEDULE;
}
/**
* Calculate next scheduled run time
* @returns {string|null} ISO timestamp or null if disabled
*/
function getNextScheduledRun() {
if (!config.SCRAPER_ENABLED || !scheduledJob) {
return null;
}
const { CronExpressionParser } = require('cron-parser');
try {
const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
tz: config.SCRAPER_TIMEZONE
});
return interval.next().toISOString();
} catch (error) {
return null;
}
}
/**
* Check if schedule runs more frequently than 1 hour
* @param {string} schedule - Cron expression
* @returns {boolean} True if schedule is too frequent
*/
function isScheduleTooFrequent(schedule) {
const parts = schedule.trim().split(/\s+/);
if (parts.length < 5) return false;
const minuteField = parts[0];
// If minute field is */N with N < 60, it runs more than once per hour
if (/^\*\/\d+$/.test(minuteField)) {
const interval = parseInt(minuteField.substring(2), 10);
if (interval < 60) return true;
}
// If minute field is *, it runs every minute
if (minuteField === '*') return true;
return false;
}
/**
* Initialize the scraper scheduler
* @param {Db} db - MongoDB database instance
*/
function initializeScheduler(db) {
const logger = createLogger('scheduler');
// Check if scheduling is enabled
if (!config.SCRAPER_ENABLED) {
logger.info('Scraper scheduling is disabled');
return;
}
// Validate cron expression
if (!cron.validate(config.SCRAPER_SCHEDULE)) {
logger.error('Invalid cron schedule expression', {
schedule: config.SCRAPER_SCHEDULE
});
logger.warn('Falling back to default schedule: 0 6 * * *');
config.SCRAPER_SCHEDULE = '0 6 * * *';
}
// Validate minimum interval (1 hour)
if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
originalSchedule: config.SCRAPER_SCHEDULE
});
config.SCRAPER_SCHEDULE = '0 * * * *';
}
// Create the scheduled job
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
const jobId = crypto.randomUUID();
const jobLogger = createLogger(jobId);
jobLogger.info('Scheduled scrape triggered');
// Check if already running or shutting down
if (!acquireLock(jobId)) {
jobLogger.warn('Skipped - scrape already in progress');
return;
}
try {
const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
runningJobPromise = jobExecution;
await jobExecution;
} catch (error) {
jobLogger.error('Scheduled scrape failed', {
errorType: error.name,
errorMessage: error.message
});
} finally {
runningJobPromise = null;
releaseLock();
}
}, {
timezone: config.SCRAPER_TIMEZONE,
scheduled: true
});
logger.info('Scraper scheduler initialized', {
schedule: config.SCRAPER_SCHEDULE,
timezone: config.SCRAPER_TIMEZONE,
nextRun: getNextScheduledRun()
});
}
/**
* Stop the scheduler (for graceful shutdown)
*/
function stopScheduler() {
if (scheduledJob) {
scheduledJob.stop();
scheduledJob = null;
}
}
/**
* Check if the scraper is in the process of shutting down
* @returns {boolean}
*/
function isShuttingDown() {
return shuttingDown;
}
/**
* Perform a graceful shutdown of the scraper
* - Stops the cron scheduler to prevent new jobs
* - Waits for any running job to complete (with timeout)
* - Releases the mutex lock
* - Logs shutdown progress
* @returns {Promise<void>}
*/
async function gracefulShutdown() {
const logger = createLogger('shutdown');
logger.info('Shutdown initiated');
// Mark as shutting down to prevent new jobs
shuttingDown = true;
// Stop the cron scheduler
stopScheduler();
// Wait for running job to complete (with timeout)
if (isRunning && runningJobPromise) {
logger.info('Waiting for running job to complete', {
jobId: currentJobId,
timeout: config.SHUTDOWN_TIMEOUT
});
let timeoutHandle;
const timeoutPromise = new Promise((resolve) => {
timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
});
const result = await Promise.race([
runningJobPromise.then(() => 'completed').catch(() => 'completed'),
timeoutPromise
]);
clearTimeout(timeoutHandle);
if (result === 'timeout') {
logger.warn('Shutdown wait for running job timed out', {
jobId: currentJobId,
timeout: config.SHUTDOWN_TIMEOUT
});
}
}
// Force release the lock
releaseLock();
logger.info('Shutdown complete');
}
/**
* Register process signal handlers for graceful shutdown
* Listens for SIGTERM and SIGINT signals
*/
function registerSignalHandlers() {
process.on('SIGTERM', () => {
gracefulShutdown();
});
process.on('SIGINT', () => {
gracefulShutdown();
});
}
module.exports = {
isScraperRunning,
getCurrentJobId,
acquireLock,
releaseLock,
getScheduleExpression,
getNextScheduledRun,
initializeScheduler,
stopScheduler,
isShuttingDown,
gracefulShutdown,
registerSignalHandlers
};

821
package-lock.json generated
View File

@ -24,9 +24,6 @@
"uuid": "^13.0.0" "uuid": "^13.0.0"
}, },
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.18.0",
"eslint": "^9.18.0",
"globals": "^15.14.0",
"jest": "^30.2.0", "jest": "^30.2.0",
"mongodb-memory-server": "^11.0.1", "mongodb-memory-server": "^11.0.1",
"nodemon": "^3.0.2", "nodemon": "^3.0.2",
@ -636,285 +633,6 @@
"tslib": "^2.4.0" "tslib": "^2.4.0"
} }
}, },
"node_modules/@eslint-community/eslint-utils": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz",
"integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"eslint-visitor-keys": "^3.4.3"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
},
"peerDependencies": {
"eslint": "^6.0.0 || ^7.0.0 || >=8.0.0"
}
},
"node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": {
"version": "3.4.3",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz",
"integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/@eslint-community/regexpp": {
"version": "4.12.2",
"resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz",
"integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^12.0.0 || ^14.0.0 || >=16.0.0"
}
},
"node_modules/@eslint/config-array": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz",
"integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@eslint/object-schema": "^2.1.7",
"debug": "^4.3.1",
"minimatch": "^3.1.2"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/config-array/node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@eslint/config-array/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@eslint/config-helpers": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz",
"integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@eslint/core": "^0.17.0"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/core": {
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz",
"integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@types/json-schema": "^7.0.15"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/eslintrc": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz",
"integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
"espree": "^10.0.1",
"globals": "^14.0.0",
"ignore": "^5.2.0",
"import-fresh": "^3.2.1",
"js-yaml": "^4.1.1",
"minimatch": "^3.1.2",
"strip-json-comments": "^3.1.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/@eslint/eslintrc/node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"dev": true,
"license": "Python-2.0"
},
"node_modules/@eslint/eslintrc/node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@eslint/eslintrc/node_modules/globals": {
"version": "14.0.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz",
"integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@eslint/eslintrc/node_modules/js-yaml": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
"dev": true,
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/@eslint/eslintrc/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@eslint/js": {
"version": "9.39.2",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz",
"integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://eslint.org/donate"
}
},
"node_modules/@eslint/object-schema": {
"version": "2.1.7",
"resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz",
"integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/plugin-kit": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz",
"integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@eslint/core": "^0.17.0",
"levn": "^0.4.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@humanfs/core": {
"version": "0.19.1",
"resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
"integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": ">=18.18.0"
}
},
"node_modules/@humanfs/node": {
"version": "0.16.7",
"resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz",
"integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@humanfs/core": "^0.19.1",
"@humanwhocodes/retry": "^0.4.0"
},
"engines": {
"node": ">=18.18.0"
}
},
"node_modules/@humanwhocodes/module-importer": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
"integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": ">=12.22"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/nzakas"
}
},
"node_modules/@humanwhocodes/retry": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz",
"integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": ">=18.18"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/nzakas"
}
},
"node_modules/@isaacs/cliui": { "node_modules/@isaacs/cliui": {
"version": "8.0.2", "version": "8.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@ -1504,13 +1222,6 @@
"@babel/types": "^7.28.2" "@babel/types": "^7.28.2"
} }
}, },
"node_modules/@types/estree": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
"integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/istanbul-lib-coverage": { "node_modules/@types/istanbul-lib-coverage": {
"version": "2.0.6", "version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz",
@ -1538,13 +1249,6 @@
"@types/istanbul-lib-report": "*" "@types/istanbul-lib-report": "*"
} }
}, },
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "25.0.10", "version": "25.0.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.10.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.10.tgz",
@ -1883,29 +1587,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/acorn": {
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"license": "MIT",
"bin": {
"acorn": "bin/acorn"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/acorn-jsx": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
"integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
"node_modules/agent-base": { "node_modules/agent-base": {
"version": "7.1.4", "version": "7.1.4",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
@ -1916,23 +1597,6 @@
"node": ">= 14" "node": ">= 14"
} }
}, },
"node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/ansi-escapes": { "node_modules/ansi-escapes": {
"version": "4.3.2", "version": "4.3.2",
"resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
@ -2904,13 +2568,6 @@
} }
} }
}, },
"node_modules/deep-is": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
"dev": true,
"license": "MIT"
},
"node_modules/deepmerge": { "node_modules/deepmerge": {
"version": "4.3.1", "version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
@ -3215,214 +2872,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/eslint": {
"version": "9.39.2",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz",
"integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/eslint-utils": "^4.8.0",
"@eslint-community/regexpp": "^4.12.1",
"@eslint/config-array": "^0.21.1",
"@eslint/config-helpers": "^0.4.2",
"@eslint/core": "^0.17.0",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "9.39.2",
"@eslint/plugin-kit": "^0.4.1",
"@humanfs/node": "^0.16.6",
"@humanwhocodes/module-importer": "^1.0.1",
"@humanwhocodes/retry": "^0.4.2",
"@types/estree": "^1.0.6",
"ajv": "^6.12.4",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.6",
"debug": "^4.3.2",
"escape-string-regexp": "^4.0.0",
"eslint-scope": "^8.4.0",
"eslint-visitor-keys": "^4.2.1",
"espree": "^10.4.0",
"esquery": "^1.5.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
"file-entry-cache": "^8.0.0",
"find-up": "^5.0.0",
"glob-parent": "^6.0.2",
"ignore": "^5.2.0",
"imurmurhash": "^0.1.4",
"is-glob": "^4.0.0",
"json-stable-stringify-without-jsonify": "^1.0.1",
"lodash.merge": "^4.6.2",
"minimatch": "^3.1.2",
"natural-compare": "^1.4.0",
"optionator": "^0.9.3"
},
"bin": {
"eslint": "bin/eslint.js"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://eslint.org/donate"
},
"peerDependencies": {
"jiti": "*"
},
"peerDependenciesMeta": {
"jiti": {
"optional": true
}
}
},
"node_modules/eslint-scope": {
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz",
"integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"esrecurse": "^4.3.0",
"estraverse": "^5.2.0"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/eslint-visitor-keys": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz",
"integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/eslint/node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/eslint/node_modules/escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/eslint/node_modules/find-up": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
"integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
"dev": true,
"license": "MIT",
"dependencies": {
"locate-path": "^6.0.0",
"path-exists": "^4.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/eslint/node_modules/glob-parent": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
"integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
"dev": true,
"license": "ISC",
"dependencies": {
"is-glob": "^4.0.3"
},
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/eslint/node_modules/locate-path": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
"integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
"dev": true,
"license": "MIT",
"dependencies": {
"p-locate": "^5.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/eslint/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/eslint/node_modules/p-locate": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
"integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
"dev": true,
"license": "MIT",
"dependencies": {
"p-limit": "^3.0.2"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/espree": {
"version": "10.4.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz",
"integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"acorn": "^8.15.0",
"acorn-jsx": "^5.3.2",
"eslint-visitor-keys": "^4.2.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/esprima": { "node_modules/esprima": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
@ -3437,52 +2886,6 @@
"node": ">=4" "node": ">=4"
} }
}, },
"node_modules/esquery": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz",
"integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
"estraverse": "^5.1.0"
},
"engines": {
"node": ">=0.10"
}
},
"node_modules/esrecurse": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
"integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"estraverse": "^5.2.0"
},
"engines": {
"node": ">=4.0"
}
},
"node_modules/estraverse": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
"integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
"dev": true,
"license": "BSD-2-Clause",
"engines": {
"node": ">=4.0"
}
},
"node_modules/esutils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
"dev": true,
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/etag": { "node_modules/etag": {
"version": "1.8.1", "version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
@ -3622,13 +3025,6 @@
"express": ">= 4.11" "express": ">= 4.11"
} }
}, },
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"dev": true,
"license": "MIT"
},
"node_modules/fast-fifo": { "node_modules/fast-fifo": {
"version": "1.3.2", "version": "1.3.2",
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
@ -3643,13 +3039,6 @@
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
"node_modules/fast-levenshtein": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
"integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
"dev": true,
"license": "MIT"
},
"node_modules/fast-safe-stringify": { "node_modules/fast-safe-stringify": {
"version": "2.1.1", "version": "2.1.1",
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
@ -3667,19 +3056,6 @@
"bser": "2.1.1" "bser": "2.1.1"
} }
}, },
"node_modules/file-entry-cache": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
"integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"flat-cache": "^4.0.0"
},
"engines": {
"node": ">=16.0.0"
}
},
"node_modules/fill-range": { "node_modules/fill-range": {
"version": "7.1.1", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@ -3769,27 +3145,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/flat-cache": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz",
"integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==",
"dev": true,
"license": "MIT",
"dependencies": {
"flatted": "^3.2.9",
"keyv": "^4.5.4"
},
"engines": {
"node": ">=16"
}
},
"node_modules/flatted": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
"integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==",
"dev": true,
"license": "ISC"
},
"node_modules/follow-redirects": { "node_modules/follow-redirects": {
"version": "1.15.11", "version": "1.15.11",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
@ -4050,19 +3405,6 @@
"url": "https://github.com/sponsors/isaacs" "url": "https://github.com/sponsors/isaacs"
} }
}, },
"node_modules/globals": {
"version": "15.15.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-15.15.0.tgz",
"integrity": "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/gopd": { "node_modules/gopd": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@ -4246,16 +3588,6 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/ignore": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/ignore-by-default": { "node_modules/ignore-by-default": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
@ -4263,33 +3595,6 @@
"dev": true, "dev": true,
"license": "ISC" "license": "ISC"
}, },
"node_modules/import-fresh": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
"integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"parent-module": "^1.0.0",
"resolve-from": "^4.0.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/import-fresh/node_modules/resolve-from": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/import-local": { "node_modules/import-local": {
"version": "3.2.0", "version": "3.2.0",
"resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz",
@ -5212,13 +4517,6 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
"dev": true,
"license": "MIT"
},
"node_modules/json-parse-even-better-errors": { "node_modules/json-parse-even-better-errors": {
"version": "2.3.1", "version": "2.3.1",
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@ -5226,20 +4524,6 @@
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
"node_modules/json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true,
"license": "MIT"
},
"node_modules/json-stable-stringify-without-jsonify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
"integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==",
"dev": true,
"license": "MIT"
},
"node_modules/json5": { "node_modules/json5": {
"version": "2.2.3", "version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
@ -5302,16 +4586,6 @@
"safe-buffer": "^5.0.1" "safe-buffer": "^5.0.1"
} }
}, },
"node_modules/keyv": {
"version": "4.5.4",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
"integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
"dev": true,
"license": "MIT",
"dependencies": {
"json-buffer": "3.0.1"
}
},
"node_modules/leven": { "node_modules/leven": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz",
@ -5322,20 +4596,6 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/levn": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
"integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"prelude-ls": "^1.2.1",
"type-check": "~0.4.0"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/lines-and-columns": { "node_modules/lines-and-columns": {
"version": "1.2.4", "version": "1.2.4",
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@ -5392,13 +4652,6 @@
"integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
"dev": true,
"license": "MIT"
},
"node_modules/lodash.once": { "node_modules/lodash.once": {
"version": "4.1.1", "version": "4.1.1",
"resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
@ -6046,24 +5299,6 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
"integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==",
"dev": true,
"license": "MIT",
"dependencies": {
"deep-is": "^0.1.3",
"fast-levenshtein": "^2.0.6",
"levn": "^0.4.1",
"prelude-ls": "^1.2.1",
"type-check": "^0.4.0",
"word-wrap": "^1.2.5"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/p-limit": { "node_modules/p-limit": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@ -6126,19 +5361,6 @@
"dev": true, "dev": true,
"license": "BlueOak-1.0.0" "license": "BlueOak-1.0.0"
}, },
"node_modules/parent-module": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
"dev": true,
"license": "MIT",
"dependencies": {
"callsites": "^3.0.0"
},
"engines": {
"node": ">=6"
}
},
"node_modules/parse-json": { "node_modules/parse-json": {
"version": "5.2.0", "version": "5.2.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz",
@ -6389,16 +5611,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/prelude-ls": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
"integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/pretty-format": { "node_modules/pretty-format": {
"version": "30.2.0", "version": "30.2.0",
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz",
@ -7304,19 +6516,6 @@
"dev": true, "dev": true,
"license": "0BSD" "license": "0BSD"
}, },
"node_modules/type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
"integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
"dev": true,
"license": "MIT",
"dependencies": {
"prelude-ls": "^1.2.1"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/type-detect": { "node_modules/type-detect": {
"version": "4.0.8", "version": "4.0.8",
"resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
@ -7457,16 +6656,6 @@
"browserslist": ">= 4.21.0" "browserslist": ">= 4.21.0"
} }
}, },
"node_modules/uri-js": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
"punycode": "^2.1.0"
}
},
"node_modules/utils-merge": { "node_modules/utils-merge": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@ -7595,16 +6784,6 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
"integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/wrap-ansi": { "node_modules/wrap-ansi": {
"version": "8.1.0", "version": "8.1.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",

7
package.json
View File

@ -8,9 +8,7 @@
"dev": "nodemon server.js", "dev": "nodemon server.js",
"test": "jest", "test": "jest",
"test:watch": "jest --watch", "test:watch": "jest --watch",
"test:coverage": "jest --coverage", "test:coverage": "jest --coverage"
"lint": "eslint .",
"lint:fix": "eslint . --fix"
}, },
"keywords": [ "keywords": [
"apartments", "apartments",
@ -36,9 +34,6 @@
"uuid": "^13.0.0" "uuid": "^13.0.0"
}, },
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.18.0",
"eslint": "^9.18.0",
"globals": "^15.14.0",
"jest": "^30.2.0", "jest": "^30.2.0",
"mongodb-memory-server": "^11.0.1", "mongodb-memory-server": "^11.0.1",
"nodemon": "^3.0.2", "nodemon": "^3.0.2",

231
routes/admin.js
View File

@ -1142,236 +1142,5 @@ router.patch('/settings', async (req, res) => {
} }
}); });
// ============================================================
// Scraper Endpoints
// ============================================================
const crypto = require('crypto');
const {
isScraperRunning,
acquireLock,
releaseLock,
getCurrentJobId,
getScheduleExpression,
getNextScheduledRun
} = require('../jobs/scraperJob');
const { runScrape } = require('../services/scraperService');
const { createLogger } = require('../services/scraperLogger');
const scraperConfig = require('../config/scraper');
// Logger for scraper admin routes
const scraperRouteLogger = createLogger('scraper-admin');
// Rate limiting for manual scrape trigger (per-user, in-memory)
const RATE_LIMIT_MAX_REQUESTS = 5;
const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
const rateLimitStore = new Map();
/**
* Check rate limit for a given user ID.
* Returns an object indicating whether the request is allowed.
*
* @param {string} userId - The user ID to check
* @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
*/
function checkRateLimit(userId) {
const now = Date.now();
const userKey = userId.toString();
if (!rateLimitStore.has(userKey)) {
rateLimitStore.set(userKey, []);
}
const timestamps = rateLimitStore.get(userKey);
// Remove timestamps outside the current window
const windowStart = now - RATE_LIMIT_WINDOW_MS;
const validTimestamps = timestamps.filter(ts => ts > windowStart);
rateLimitStore.set(userKey, validTimestamps);
if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
// Calculate when the oldest request in the window will expire
const oldestTimestamp = validTimestamps[0];
const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
return { allowed: false, retryAfterSeconds };
}
// Record this request
validTimestamps.push(now);
return { allowed: true, retryAfterSeconds: null };
}
/**
* Reset the rate limiter (for testing)
*/
function resetRateLimiter() {
rateLimitStore.clear();
}
/**
* POST /api/admin/scraper/run
* Trigger a manual scrape
*
* Request body (optional):
* - dryRun: boolean - Skip database writes for safe testing
* - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
*/
router.post('/scraper/run', async (req, res) => {
try {
const db = req.app.locals.db;
const { dryRun = false, htmlContent = null } = req.body || {};
// Check rate limit (per-user, before mutex check)
const rateLimitResult = checkRateLimit(req.user._id);
if (!rateLimitResult.allowed) {
res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
return res.status(429).json({
error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
});
}
// Check if scraper is already running
if (isScraperRunning()) {
return res.status(409).json({ error: 'Scrape already in progress' });
}
// Generate job ID and acquire lock
const jobId = crypto.randomUUID();
acquireLock(jobId);
// Log admin action
await logActivity(db, {
userId: req.user._id.toString(),
action: 'ADMIN_TRIGGER_SCRAPE',
metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
});
// Start scrape asynchronously (do not await - return 202 immediately)
runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
.catch((error) => {
scraperRouteLogger.error('Async scrape failed', {
errorType: error.name,
errorMessage: error.message
});
})
.finally(() => releaseLock());
// Return immediately with job ID
res.status(202).json({
data: {
jobId,
status: 'started',
dryRun,
message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
}
});
} catch (error) {
scraperRouteLogger.error('Error triggering scrape', {
errorType: error.name,
errorMessage: error.message
});
res.status(500).json({ error: 'Failed to start scrape job' });
}
});
/**
* GET /api/admin/scraper/status
* Get current scraper status including running state, last run details,
* next scheduled run, and schedule expression.
*/
router.get('/scraper/status', async (req, res) => {
try {
const db = req.app.locals.db;
// Get last run from history
const lastRunDoc = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.findOne({}, { sort: { startedAt: -1 } });
const lastRun = lastRunDoc ? {
jobId: lastRunDoc.jobId,
timestamp: lastRunDoc.startedAt,
status: lastRunDoc.status,
duration: lastRunDoc.duration,
trigger: lastRunDoc.trigger,
unitsProcessed: lastRunDoc.unitsProcessed,
pricesInserted: lastRunDoc.pricesInserted,
errors: lastRunDoc.errors?.length > 0 ? lastRunDoc.errors : null
} : null;
res.json({
data: {
currentStatus: isScraperRunning() ? 'running' : 'idle',
runningJobId: isScraperRunning() ? getCurrentJobId() : null,
lastRun,
nextScheduledRun: getNextScheduledRun(),
schedule: getScheduleExpression()
}
});
} catch (error) {
scraperRouteLogger.error('Error fetching scraper status', {
errorType: error.name,
errorMessage: error.message
});
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
/**
* GET /api/admin/scraper/history
* Get scraper run history with pagination
*
* Query params:
* - limit: Number of records (1-100, default 30)
* - offset: Number of records to skip (default 0)
*/
router.get('/scraper/history', async (req, res) => {
try {
const db = req.app.locals.db;
// Parse and validate pagination parameters
let limit = parseInt(req.query.limit);
let offset = parseInt(req.query.offset);
// Validate limit
if (req.query.limit !== undefined) {
if (isNaN(limit) || limit < 1) {
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
}
limit = Math.min(limit, 100);
} else {
limit = 30;
}
// Validate offset
if (req.query.offset !== undefined) {
if (isNaN(offset) || offset < 0) {
return res.status(400).json({ error: 'Invalid offset parameter' });
}
} else {
offset = 0;
}
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
.find({})
.sort({ startedAt: -1 })
.skip(offset)
.limit(limit)
.toArray();
res.json({ data: history });
} catch (error) {
scraperRouteLogger.error('Error fetching scraper history', {
errorType: error.name,
errorMessage: error.message
});
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
module.exports = router; module.exports = router;
module.exports.clearStatsCache = clearStatsCache; module.exports.clearStatsCache = clearStatsCache;
module.exports.resetRateLimiter = resetRateLimiter;

35
server.js
View File

@ -11,9 +11,6 @@ const activityRoutes = require('./routes/activity');
const adminRoutes = require('./routes/admin'); const adminRoutes = require('./routes/admin');
const { createIndexes } = require('./models/user'); const { createIndexes } = require('./models/user');
const { createActivityIndexes } = require('./services/activityLogger'); const { createActivityIndexes } = require('./services/activityLogger');
const { createScraperIndexes } = require('./services/scraperService');
const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
const scraperConfig = require('./config/scraper');
const app = express(); const app = express();
const PORT = process.env.PORT || 3000; const PORT = process.env.PORT || 3000;
@ -83,32 +80,14 @@ async function connectToMongoDB() {
await createIndexes(db); await createIndexes(db);
await createActivityIndexes(db); await createActivityIndexes(db);
console.log('✅ Passport configured and indexes created'); console.log('✅ Passport configured and indexes created');
// Initialize scraper (indexes + scheduler) - errors logged but don't crash server
try {
const { createLogger } = require('./services/scraperLogger');
const scraperLogger = createLogger('server-init');
await createScraperIndexes(db, scraperLogger);
if (scraperConfig.SCRAPER_ENABLED) {
initializeScheduler(db);
console.log('✅ Scraper scheduler initialized');
} else {
console.log('ℹ️ Scraper scheduling disabled');
}
} catch (error) {
console.error('⚠️ Scraper initialization failed (server continues):', error.message);
}
} catch (error) { } catch (error) {
console.error('❌ Failed to connect to MongoDB:', error); console.error('❌ Failed to connect to MongoDB:', error);
process.exit(1); process.exit(1);
} }
} }
// Register scraper signal handlers for graceful shutdown // Helper function to get today's date in UTC
registerSignalHandlers(); const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
// Helper function to get today's date in configured timezone
const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
// Cache for the most recent date with data // Cache for the most recent date with data
let cachedLatestDate = null; let cachedLatestDate = null;
@ -143,7 +122,7 @@ const getLatestDateWithData = async () => {
} }
// Fallback to UTC today if no data found // Fallback to UTC today if no data found
return getTodayDate(); return getTodayDateUTC();
}; };
// Helper function to get yesterday relative to the latest date with data // Helper function to get yesterday relative to the latest date with data
@ -641,10 +620,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
]).toArray(); ]).toArray();
// 4. Historical Comparison - current vs 30/90/365 day averages // 4. Historical Comparison - current vs 30/90/365 day averages
const tz = scraperConfig.SCRAPER_TIMEZONE; const now = new Date();
const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz }); const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const [currentAvg, avg30, avg90, avg365] = await Promise.all([ const [currentAvg, avg30, avg90, avg365] = await Promise.all([
db.collection(PRICES_COLLECTION).aggregate([ db.collection(PRICES_COLLECTION).aggregate([

920
services/scraperService.js
View File

@ -1,920 +0,0 @@
/**
* Scraper Service
*
* Core scraper logic including HTTP fetching, HTML parsing,
* data transformation, and database operations.
*/
const axios = require('axios');
const cheerio = require('cheerio');
const crypto = require('crypto');
const config = require('../config/scraper');
const { createLogger } = require('./scraperLogger');
/**
* Sleep utility for retry delays
* @param {number} ms - Milliseconds to sleep
* @returns {Promise<void>}
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Determine if an error is retryable
* @param {Error} error - Axios error
* @returns {boolean} True if should retry
*/
function isRetryableError(error) {
// Network errors (timeout, DNS, connection) don't have a response property
if (!error.response) {
return true;
}
const status = error.response.status;
// 5xx server errors are retryable
if (status >= 500) {
return true;
}
// 429 Too Many Requests - do not retry immediately
if (status === 429) {
return false;
}
// 4xx client errors - do not retry
return false;
}
/**
* Fetch page HTML with retry logic
* @param {string} url - Target URL
* @param {Object} logger - Logger instance
* @returns {Promise<string>} HTML content
* @throws {Error} After all retries exhausted
*/
async function fetchPage(url, logger) {
const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
let lastError;
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
try {
logger.info('Fetching page', { url, attempt });
const response = await axios.get(url, {
timeout,
headers: {
'User-Agent': config.USER_AGENT
},
maxRedirects: 5,
validateStatus: (status) => status < 400 // Accept 2xx and 3xx
});
logger.info('Page fetched successfully', {
status: response.status,
contentLength: response.data.length
});
return response.data;
} catch (error) {
lastError = error;
// Determine if error is retryable
const isRetryable = isRetryableError(error);
logger.warn('Fetch attempt failed', {
attempt,
errorType: error.name,
errorMessage: error.message,
statusCode: error.response?.status,
isRetryable
});
// Don't retry non-retryable errors (4xx)
if (!isRetryable) {
throw error;
}
// Don't wait after last attempt
if (attempt <= maxRetries) {
const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
logger.info('Waiting before retry', { delay });
await sleep(delay);
}
}
}
throw lastError;
}
/**
* Parse unit data from HTML using cheerio
* @param {string} html - HTML content
* @param {Object} logger - Logger instance
* @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
*/
function parseUnits(html, logger) {
const $ = cheerio.load(html);
const units = [];
// Find the main container
const container = $('section.spaces__tab-unit');
if (container.length === 0) {
logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
return [];
}
// Extract each article element
container.find('article').each((index, article) => {
const $article = $(article);
const unit = {
// Core identifiers
id: $article.attr('data-spaces-id'),
unit_code: $article.attr('data-spaces-unit'),
unit_id: $article.attr('data-spaces-unit-id'),
// Physical attributes
floor: $article.attr('data-spaces-unit-floor'),
area: $article.attr('data-spaces-sort-area'),
bed_count: $article.attr('data-spaces-sort-bed'),
bath_count: $article.attr('data-spaces-sort-bath'),
// Pricing
price: $article.attr('data-spaces-sort-price'),
// Availability
available: $article.attr('data-spaces-available'),
unavailable: $article.attr('data-spaces-unavailable'),
soonest: $article.attr('data-spaces-soonest'),
date_available: $article.attr('data-spaces-sort-date'),
// Plan information
plan_id: $article.attr('data-spaces-plan-id'),
plan_name: $article.attr('data-spaces-sort-plan-name'),
// Property information
obj_type: $article.attr('data-spaces-obj'),
community: $article.attr('data-spaces-community'),
asset: $article.attr('data-spaces-asset'),
// URLs
href: $article.attr('data-spaces-href'),
inventory_href: $article.attr('data-spaces-inventory-href'),
// Specials
specials_content: $article.attr('data-spaces-specials-content')
};
// Extract image URL from nested element if present
const imgElement = $article.find('img').first();
if (imgElement.length > 0) {
unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
}
units.push(unit);
});
logger.info('Units parsed', { count: units.length });
// Deduplicate by unit_code (in case of duplicate articles)
const seen = new Set();
const deduplicated = units.filter(unit => {
if (!unit.unit_code || seen.has(unit.unit_code)) {
return false;
}
seen.add(unit.unit_code);
return true;
});
if (deduplicated.length < units.length) {
logger.warn('Duplicate units removed', {
original: units.length,
deduplicated: deduplicated.length
});
}
return deduplicated;
}
// ============================================================
// Data Type Conversion Helpers
// ============================================================
/**
* Parse string to integer, return null for invalid
* @param {*} value - Value to parse
* @returns {number|null} Parsed integer or null
*/
function parseInteger(value) {
if (value === null || value === undefined || value === '') {
return null;
}
const parsed = parseInt(value, 10);
return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
}
/**
* Parse string to positive integer, return null for 0 or invalid.
* Used for fields like area where 0 means "not available".
* @param {*} value - Value to parse
* @returns {number|null} Parsed positive integer or null
*/
function parsePositiveInteger(value) {
const parsed = parseInteger(value);
return parsed === 0 ? null : parsed;
}
/**
* Parse value that could be integer or string identifier.
* Returns integer if cleanly parseable, otherwise trimmed string.
* @param {*} value - Value to parse
* @returns {number|string|null} Parsed integer, trimmed string, or null
*/
function parseIntegerOrString(value) {
if (value === null || value === undefined || value === '') {
return null;
}
const parsed = parseInt(value, 10);
// If it parses cleanly to an integer, return integer
if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
return parsed;
}
// Otherwise return as trimmed string
return String(value).trim();
}
/**
* Parse price, stripping non-numeric characters.
* Handles "$1,234" format and "Call for pricing" text.
* @param {*} value - Value to parse
* @returns {number|null} Parsed price or null
*/
function parsePrice(value) {
if (value === null || value === undefined || value === '') {
return null;
}
// Check for "Call for pricing" or similar text
if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
return null;
}
// Strip non-numeric characters except decimal point
const cleaned = String(value).replace(/[^0-9.]/g, '');
const parsed = parseInt(cleaned, 10);
if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
return null;
}
// Negative prices are invalid
if (parsed < 0) {
return null;
}
return parsed;
}
/**
* Parse boolean from string.
* Handles "true"/"false"/"1"/"0" and actual boolean values.
* @param {*} value - Value to parse
* @returns {boolean|null} Parsed boolean or null
*/
function parseBoolean(value) {
if (value === null || value === undefined || value === '') {
return null;
}
if (typeof value === 'boolean') {
return value;
}
const str = String(value).toLowerCase().trim();
if (str === 'true' || str === '1') return true;
if (str === 'false' || str === '0') return false;
return null;
}
/**
* Parse float value.
* Named parseFloatValue to avoid shadowing the global parseFloat.
* @param {*} value - Value to parse
* @returns {number|null} Parsed float or null
*/
function parseFloatValue(value) {
if (value === null || value === undefined || value === '') {
return null;
}
const parsed = Number.parseFloat(value);
return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
}
/**
* Trim string, return null for empty.
* Converts non-string values to string before trimming.
* @param {*} value - Value to trim
* @returns {string|null} Trimmed string or null
*/
function trimString(value) {
if (value === null || value === undefined) {
return null;
}
const trimmed = String(value).trim();
return trimmed === '' ? null : trimmed;
}
// ============================================================
// Main Data Type Conversion Function
// ============================================================
/**
* Convert unit data types from strings to proper types.
* Applies the appropriate parser to each field based on its expected type.
* @param {Object} unit - Raw unit object with string values
* @returns {Object} Unit object with converted types
*/
function convertDataTypes(unit) {
return {
// Integer fields
id: parseInteger(unit.id),
unit_id: parseIntegerOrString(unit.unit_id),
floor: parseInteger(unit.floor),
area: parsePositiveInteger(unit.area),
bed_count: parseInteger(unit.bed_count),
plan_id: parseInteger(unit.plan_id),
asset: parseInteger(unit.asset),
date_available: parseInteger(unit.date_available),
// Float fields
bath_count: parseFloatValue(unit.bath_count),
// Price field (special handling)
price: parsePrice(unit.price),
// Boolean fields
available: parseBoolean(unit.available),
unavailable: parseBoolean(unit.unavailable),
// String fields (trim and preserve)
unit_code: trimString(unit.unit_code),
plan_name: trimString(unit.plan_name),
soonest: trimString(unit.soonest),
obj_type: trimString(unit.obj_type),
community: trimString(unit.community),
href: trimString(unit.href),
inventory_href: trimString(unit.inventory_href),
image_url: trimString(unit.image_url),
specials_content: trimString(unit.specials_content)
};
}
// ============================================================
// Date Helpers
// ============================================================
/**
* Calculate yesterday's date from a given date string.
* @param {string} dateStr - Date in YYYY-MM-DD format
* @returns {string} Yesterday's date in YYYY-MM-DD format
*/
function getYesterday(dateStr) {
const date = new Date(dateStr + 'T00:00:00Z');
date.setUTCDate(date.getUTCDate() - 1);
return date.toISOString().split('T')[0];
}
// ============================================================
// Database Operations
// ============================================================
/**
* Upsert unit records to database using bulkWrite.
* Each unit is matched by unit_code as the unique key.
* Sets last_scraped and data_source on every update.
* Sets first_seen only on initial insert via $setOnInsert.
*
* @param {Db} db - MongoDB database instance
* @param {Array<Object>} units - Array of unit objects
* @param {Object} logger - Logger instance
* @returns {Promise<Object>} Bulk write result
*/
async function upsertUnits(db, units, logger) {
const collection = db.collection(config.COLLECTIONS.UNITS);
const now = new Date().toISOString();
const operations = units.map(unit => ({
updateOne: {
filter: { unit_code: unit.unit_code },
update: {
$set: {
...unit,
last_scraped: now,
data_source: 'web_scraper'
},
$setOnInsert: {
first_seen: now
}
},
upsert: true
}
}));
if (operations.length === 0) {
logger.warn('No units to upsert');
return { modifiedCount: 0, upsertedCount: 0 };
}
try {
const result = await collection.bulkWrite(operations, { ordered: false });
logger.info('Units upserted', {
matched: result.matchedCount,
modified: result.modifiedCount,
upserted: result.upsertedCount
});
return result;
} catch (error) {
logger.error('Failed to upsert units', {
errorType: error.name,
errorMessage: error.message
});
throw error;
}
}
/**
* Insert price records for today using bulkWrite with upsert.
* Uses unit_code + date_checked as the composite key to prevent
* duplicate price records for the same unit on the same day.
* Re-running on the same day updates existing records (idempotent).
*
* @param {Db} db - MongoDB database instance
* @param {Array<Object>} units - Array of unit objects
* @param {string} date - Date in YYYY-MM-DD format
* @param {Object} logger - Logger instance
* @returns {Promise<Object>} Insert result with insertedCount
*/
async function insertPrices(db, units, date, logger) {
const collection = db.collection(config.COLLECTIONS.PRICES);
const now = new Date().toISOString();
// Filter out units without prices
const unitsWithPrices = units.filter(u => u.price !== null);
const priceRecords = unitsWithPrices.map(unit => ({
unit_code: unit.unit_code,
date_checked: date,
price: unit.price,
last_updated: now,
data_source: 'web_scraper'
}));
if (priceRecords.length === 0) {
logger.warn('No price records to insert');
return { insertedCount: 0 };
}
// Use updateOne with upsert to handle re-runs on same day
const operations = priceRecords.map(record => ({
updateOne: {
filter: {
unit_code: record.unit_code,
date_checked: record.date_checked
},
update: { $set: record },
upsert: true
}
}));
try {
const result = await collection.bulkWrite(operations, { ordered: false });
logger.info('Prices inserted', {
inserted: result.upsertedCount,
updated: result.modifiedCount
});
return { insertedCount: result.upsertedCount + result.modifiedCount };
} catch (error) {
logger.error('Failed to insert prices', {
errorType: error.name,
errorMessage: error.message
});
throw error;
}
}
/**
* Mark units not in current scrape as stale/unavailable.
* Uses updateMany to set available: false and marked_stale_date
* for all units whose unit_code is NOT in the current scrape
* and that are currently available.
*
* @param {Db} db - MongoDB database instance
* @param {Set<string>} currentUnitCodes - Unit codes from current scrape
* @param {string} date - Current date in YYYY-MM-DD format
* @param {Object} logger - Logger instance
* @returns {Promise<Object>} Update result
*/
async function markStaleUnits(db, currentUnitCodes, date, logger) {
const collection = db.collection(config.COLLECTIONS.UNITS);
try {
const result = await collection.updateMany(
{
unit_code: { $nin: [...currentUnitCodes] },
available: true
},
{
$set: {
available: false,
marked_stale_date: date
}
}
);
logger.info('Stale units marked', { count: result.modifiedCount });
return result;
} catch (error) {
logger.error('Failed to mark stale units', {
errorType: error.name,
errorMessage: error.message
});
throw error;
}
}
/**
* Update daily summary document.
* Upserts by date field (YYYY-MM-DD format).
* Fetches yesterday's summary for comparison metrics.
* Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
*
* @param {Db} db - MongoDB database instance
* @param {Object} summaryData - Summary data
* @param {string} summaryData.date - Date in YYYY-MM-DD format
* @param {Array<string>} summaryData.newUnits - Unit codes added today
* @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
* @param {number} summaryData.staleUnitsCount - Count of stale units
* @param {number} summaryData.totalAvailable - Total available units today
* @param {Object} logger - Logger instance
* @returns {Promise<Object>} MongoDB updateOne result
*/
async function updateDailySummary(db, summaryData, logger) {
const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
// Calculate yesterday's date for comparison
const yesterday = getYesterday(date);
// Get yesterday's summary for comparison
const yesterdaySummary = await collection.findOne({ date: yesterday });
const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
const summary = {
date,
timestamp: new Date().toISOString(),
new_units: newUnits,
rented_units: rentedUnits,
stale_units: [], // Stale units list is not tracked per PRD
new_units_count: newUnits.length,
rented_units_count: rentedUnits.length,
stale_units_count: staleUnitsCount,
net_change: newUnits.length - rentedUnits.length,
total_available_today: totalAvailable,
total_available_yesterday: yesterdayTotal,
turnover_rate: yesterdayTotal > 0
? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
: 0
};
try {
const result = await collection.updateOne(
{ date },
{ $set: summary },
{ upsert: true }
);
logger.info('Daily summary updated', {
date,
newUnits: newUnits.length,
rentedUnits: rentedUnits.length,
totalAvailable
});
return result;
} catch (error) {
logger.error('Failed to update daily summary', {
errorType: error.name,
errorMessage: error.message
});
throw error;
}
}
// ============================================================
// Additional Helpers for runScrape Orchestration
// ============================================================
/**
* Get today's date in YYYY-MM-DD format using the configured timezone.
* @returns {string} Today's date string
*/
function getToday() {
return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
}
/**
* Get the set of unit codes that had price records yesterday.
* Used to calculate new and rented units by comparison.
* @param {Db} db - MongoDB database instance
* @param {string} today - Today's date in YYYY-MM-DD format
* @returns {Promise<Set<string>>} Set of unit codes from yesterday
*/
async function getYesterdayUnitCodes(db, today) {
const yesterday = getYesterday(today);
const collection = db.collection(config.COLLECTIONS.PRICES);
const yesterdayRecords = await collection
.find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
.toArray();
return new Set(yesterdayRecords.map(r => r.unit_code));
}
// ============================================================
// Error Sanitization
// ============================================================
/**
* Sanitize an error object to remove sensitive information before storage.
* Removes file paths, connection strings, and credential patterns while
* preserving the error type and a useful general description for debugging.
*
* @param {Error} error - Error object to sanitize
* @returns {Object} Sanitized error with name, message, and optionally stack
*/
function sanitizeError(error) {
const sanitized = {
name: error.name || 'Error',
message: sanitizeMessage(error.message || ''),
};
if (error.stack) {
sanitized.stack = sanitizeMessage(error.stack);
}
return sanitized;
}
/**
* Sanitize a string message by removing sensitive patterns.
* @param {string} message - Raw error message
* @returns {string} Sanitized message
*/
function sanitizeMessage(message) {
let result = message;
// Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
// Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
// Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
// Remove Windows-style absolute paths (C:\..., D:\...)
result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
return result;
}
// ============================================================
// Scraper Run History
// ============================================================
/**
* Record scraper run to history collection.
* Called in the finally block of runScrape() to persist run metadata.
* This function must NOT throw errors - history recording should never break the scraper.
*
* @param {Db} db - MongoDB database instance
* @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
* @param {Object} logger - Logger instance
* @returns {Promise<Object|null>} Insert result, or null on failure
*/
async function recordScraperRun(db, runData, logger) {
try {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
const result = await collection.insertOne({
...runData,
recordedAt: new Date()
});
return result;
} catch (error) {
// Log but don't throw - recording history should not break scraper
logger.error('Failed to record scraper run', { errorMessage: error.message });
return null;
}
}
// ============================================================
// Index Management
// ============================================================
/**
* Create indexes on the scraper_runs collection.
* Should be called once during application startup.
* Idempotent - safe to call multiple times.
*
* @param {Db} db - MongoDB database instance
* @param {Object} logger - Logger instance
* @returns {Promise<void>}
*/
async function createScraperIndexes(db, logger) {
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
// Compound index for querying runs by status sorted by most recent
await collection.createIndex(
{ status: 1, startedAt: -1 },
{ name: 'status_startedAt' }
);
// Index for sorting all runs by start time (most recent first)
await collection.createIndex(
{ startedAt: -1 },
{ name: 'startedAt_desc' }
);
logger.info('Scraper indexes created successfully');
}
// ============================================================
// Main Orchestration Function
// ============================================================
/**
* Execute a complete scrape operation.
* Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
*
* @param {Db} db - MongoDB database instance
* @param {Object} options - Scrape options
* @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
* @param {string} [options.jobId] - Optional job ID (generated if not provided)
* @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
* @param {string} [options.htmlContent] - Use provided HTML instead of fetching
* @returns {Promise<Object>} Scrape result with status and metrics
*/
async function runScrape(db, options = {}) {
const jobId = options.jobId || crypto.randomUUID();
const trigger = options.trigger || 'manual';
const dryRun = options.dryRun || false;
const htmlContent = options.htmlContent || null;
const logger = createLogger(jobId);
const startTime = Date.now();
let result = {
jobId,
trigger,
dryRun,
status: 'running',
startedAt: new Date().toISOString(),
completedAt: null,
duration: null,
unitsProcessed: 0,
pricesInserted: 0,
newUnitsCount: 0,
rentedUnitsCount: 0,
staleUnitsCount: 0,
errors: []
};
try {
logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
// Step 1: Fetch HTML (or use provided content for testing)
const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
// Step 2: Parse units
const rawUnits = parseUnits(html, logger);
if (rawUnits.length === 0) {
logger.warn('No units found in HTML - possible structure change');
result.errors.push('No units found in HTML');
}
// Step 3: Convert data types
const units = rawUnits.map(unit => convertDataTypes(unit));
result.unitsProcessed = units.length;
// Step 4: Database operations
const today = getToday();
// Get yesterday's unit codes for comparison
const yesterdayUnits = await getYesterdayUnitCodes(db, today);
// Determine new and rented units
const currentUnitCodes = new Set(units.map(u => u.unit_code));
const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
result.newUnitsCount = newUnits.length;
result.rentedUnitsCount = rentedUnits.length;
// Database operations (skip if dryRun)
if (dryRun) {
logger.info('Dry run mode - skipping database writes', {
wouldUpsert: units.length,
wouldInsertPrices: units.filter(u => u.price !== null).length
});
result.pricesInserted = 0;
result.staleUnitsCount = 0;
} else {
// Upsert units
await upsertUnits(db, units, logger);
// Insert prices
const pricesResult = await insertPrices(db, units, today, logger);
result.pricesInserted = pricesResult.insertedCount;
// Mark stale units
const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
result.staleUnitsCount = staleResult.modifiedCount;
// Update daily summary
await updateDailySummary(db, {
date: today,
newUnits: newUnits.map(u => u.unit_code),
rentedUnits,
staleUnitsCount: result.staleUnitsCount,
totalAvailable: units.length
}, logger);
}
result.status = 'success';
} catch (error) {
const cleanError = sanitizeError(error);
logger.error('Scrape failed', {
errorType: cleanError.name,
errorMessage: cleanError.message
});
result.status = 'failed';
result.errors.push(cleanError.message);
} finally {
result.completedAt = new Date().toISOString();
result.duration = Date.now() - startTime;
// Record run to history (always runs, even on failure)
await recordScraperRun(db, result, logger);
logger.info('Scrape completed', {
status: result.status,
duration: result.duration,
unitsProcessed: result.unitsProcessed,
pricesInserted: result.pricesInserted
});
}
return result;
}
module.exports = {
runScrape,
fetchPage,
parseUnits,
convertDataTypes,
upsertUnits,
insertPrices,
markStaleUnits,
updateDailySummary,
recordScraperRun,
createScraperIndexes,
// Export helpers for testing
getToday,
getYesterdayUnitCodes,
getYesterday,
parseInteger,
parsePositiveInteger,
parseIntegerOrString,
parsePrice,
parseBoolean,
parseFloatValue,
trimString,
isRetryableError,
sleep,
sanitizeError
};