Compare commits
5 Commits
main
...
fd56561aed
| Author | SHA1 | Date | |
|---|---|---|---|
| fd56561aed | |||
| 40cd0e35bf | |||
| c23afdf0b7 | |||
| dbd8d81668 | |||
| 39bd9b1e71 |
408
.github/workflows/deploy.yml
vendored
408
.github/workflows/deploy.yml
vendored
@ -1,392 +1,60 @@
|
||||
name: CI/CD Pipeline - Apartment API
|
||||
name: Deploy Apartment API
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
NODE_VERSION: '20'
|
||||
IMAGE_NAME: apartment-api
|
||||
|
||||
jobs:
|
||||
# ============================================================
|
||||
# CI Job - Lint + Test in a single job (one checkout, one npm ci)
|
||||
# ============================================================
|
||||
ci:
|
||||
name: Lint & Test
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
outputs:
|
||||
lint_status: ${{ steps.lint.outcome }}
|
||||
lint_output: ${{ steps.lint.outputs.lint_output }}
|
||||
test_status: ${{ steps.test.outcome }}
|
||||
test_output: ${{ steps.test.outputs.test_output }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run ESLint
|
||||
id: lint
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set +e
|
||||
OUTPUT=$(npm run lint 2>&1)
|
||||
EXIT_CODE=$?
|
||||
echo "$OUTPUT"
|
||||
# Truncate before writing to GITHUB_OUTPUT to prevent
|
||||
# "argument list too long" in downstream jobs
|
||||
echo "lint_output<<EOF" >> $GITHUB_OUTPUT
|
||||
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
exit $EXIT_CODE
|
||||
|
||||
- name: Run tests
|
||||
id: test
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set +e
|
||||
OUTPUT=$(npm test -- --runInBand 2>&1)
|
||||
EXIT_CODE=$?
|
||||
echo "$OUTPUT"
|
||||
# Truncate before writing to GITHUB_OUTPUT to prevent
|
||||
# "argument list too long" in downstream jobs
|
||||
echo "test_output<<EOF" >> $GITHUB_OUTPUT
|
||||
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
exit $EXIT_CODE
|
||||
env:
|
||||
JWT_SECRET: test-jwt-secret-for-ci
|
||||
NODE_ENV: test
|
||||
|
||||
# ============================================================
|
||||
# Notify Job - Send results to n8n webhook
|
||||
# ============================================================
|
||||
notify:
|
||||
name: Send Webhook Notification
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ci]
|
||||
if: always() && github.ref != 'refs/heads/main'
|
||||
|
||||
steps:
|
||||
- name: Send results to n8n webhook
|
||||
env:
|
||||
LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
|
||||
TEST_STATUS: ${{ needs.ci.outputs.test_status }}
|
||||
RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
|
||||
RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
GH_BRANCH: ${{ github.head_ref || github.ref_name }}
|
||||
GH_SHA: ${{ github.sha }}
|
||||
GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
|
||||
GH_ACTOR: ${{ github.actor }}
|
||||
GH_EVENT: ${{ github.event_name }}
|
||||
GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
|
||||
GH_RUN_ID: ${{ github.run_id }}
|
||||
GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
|
||||
run: |
|
||||
# Determine overall status
|
||||
if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
|
||||
OVERALL_STATUS="success"
|
||||
else
|
||||
OVERALL_STATUS="failure"
|
||||
fi
|
||||
|
||||
# Outputs are already truncated at the source (ci job)
|
||||
LINT_OUTPUT="$RAW_LINT_OUTPUT"
|
||||
TEST_OUTPUT="$RAW_TEST_OUTPUT"
|
||||
|
||||
# Build JSON payload
|
||||
PAYLOAD=$(jq -n \
|
||||
--arg repo "$GH_REPO" \
|
||||
--arg branch "$GH_BRANCH" \
|
||||
--arg commit "$GH_SHA" \
|
||||
--arg commit_short "${GH_SHA:0:7}" \
|
||||
--arg commit_message "$GH_COMMIT_MSG" \
|
||||
--arg author "$GH_ACTOR" \
|
||||
--arg event "$GH_EVENT" \
|
||||
--arg pr_number "$GH_PR_NUMBER" \
|
||||
--arg run_id "$GH_RUN_ID" \
|
||||
--arg run_url "$GH_RUN_URL" \
|
||||
--arg overall_status "$OVERALL_STATUS" \
|
||||
--arg lint_status "$LINT_STATUS" \
|
||||
--arg lint_output "$LINT_OUTPUT" \
|
||||
--arg test_status "$TEST_STATUS" \
|
||||
--arg test_output "$TEST_OUTPUT" \
|
||||
--arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
'{
|
||||
repository: $repo,
|
||||
branch: $branch,
|
||||
pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
|
||||
commit: {
|
||||
sha: $commit,
|
||||
short_sha: $commit_short,
|
||||
message: $commit_message,
|
||||
author: $author
|
||||
},
|
||||
event: $event,
|
||||
run: {
|
||||
id: $run_id,
|
||||
url: $run_url
|
||||
},
|
||||
status: $overall_status,
|
||||
results: {
|
||||
lint: {
|
||||
status: $lint_status,
|
||||
output: (if $lint_status != "success" then $lint_output else null end)
|
||||
},
|
||||
test: {
|
||||
status: $test_status,
|
||||
output: (if $test_status != "success" then $test_output else null end)
|
||||
}
|
||||
},
|
||||
timestamp: $timestamp
|
||||
}')
|
||||
|
||||
# Send webhook
|
||||
curl -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD" \
|
||||
"$WEBHOOK_URL" \
|
||||
--fail --silent --show-error
|
||||
|
||||
- name: Fail if lint or tests failed
|
||||
if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
|
||||
run: |
|
||||
echo "❌ Pipeline failed:"
|
||||
echo " Lint: ${{ needs.ci.outputs.lint_status }}"
|
||||
echo " Test: ${{ needs.ci.outputs.test_status }}"
|
||||
exit 1
|
||||
|
||||
# ============================================================
|
||||
# Dependency Scan Job - Runs in parallel with tests
|
||||
# ============================================================
|
||||
scan-deps:
|
||||
name: Scan Dependencies
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Trivy
|
||||
run: |
|
||||
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
|
||||
|
||||
- name: Scan dependencies for vulnerabilities
|
||||
run: |
|
||||
trivy fs \
|
||||
--exit-code 1 \
|
||||
--ignore-unfixed \
|
||||
--severity CRITICAL,HIGH \
|
||||
--scanners vuln \
|
||||
.
|
||||
|
||||
# ============================================================
|
||||
# Build & Push Job - Build image and push to Harbor
|
||||
# ============================================================
|
||||
build:
|
||||
name: Build & Push Image
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ci, scan-deps]
|
||||
if: >-
|
||||
github.ref == 'refs/heads/main' &&
|
||||
github.event_name != 'pull_request' &&
|
||||
needs.ci.outputs.lint_status == 'success' &&
|
||||
needs.ci.outputs.test_status == 'success'
|
||||
|
||||
outputs:
|
||||
image_tag: ${{ steps.set-tag.outputs.tag }}
|
||||
full_image: ${{ steps.set-tag.outputs.full_image }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set image tag
|
||||
id: set-tag
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
echo "tag=${SHORT_SHA}" >> $GITHUB_OUTPUT
|
||||
echo "full_image=${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${SHORT_SHA}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Harbor
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ secrets.HARBOR_REGISTRY }}
|
||||
username: ${{ secrets.HARBOR_USERNAME }}
|
||||
password: ${{ secrets.HARBOR_PASSWORD }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build-push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
|
||||
${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:latest
|
||||
|
||||
# ============================================================
|
||||
# Vulnerability Scanning with Trivy
|
||||
# ============================================================
|
||||
- name: Install Trivy
|
||||
run: |
|
||||
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
run: |
|
||||
trivy image \
|
||||
--exit-code 1 \
|
||||
--ignore-unfixed \
|
||||
--vuln-type os,library \
|
||||
--severity CRITICAL,HIGH \
|
||||
--format table \
|
||||
${{ steps.set-tag.outputs.full_image }}
|
||||
|
||||
- name: Run Trivy and output SARIF
|
||||
if: always()
|
||||
run: |
|
||||
trivy image \
|
||||
--format sarif \
|
||||
--output trivy-results.sarif \
|
||||
${{ steps.set-tag.outputs.full_image }} || true
|
||||
|
||||
# ============================================================
|
||||
# SBOM Generation with Syft
|
||||
# ============================================================
|
||||
- name: Install Syft
|
||||
run: |
|
||||
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
|
||||
|
||||
- name: Generate SBOM with Syft
|
||||
run: |
|
||||
syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json
|
||||
|
||||
# ============================================================
|
||||
# Image Signing with Cosign
|
||||
# ============================================================
|
||||
- name: Install Cosign
|
||||
uses: sigstore/cosign-installer@v3
|
||||
|
||||
- name: Sign image with Cosign
|
||||
env:
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
run: |
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY \
|
||||
--yes \
|
||||
${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
|
||||
|
||||
# ============================================================
|
||||
# Attach SBOM to image in Harbor
|
||||
# ============================================================
|
||||
- name: Attach SBOM to image
|
||||
env:
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
run: |
|
||||
cosign attest --key env://COSIGN_PRIVATE_KEY \
|
||||
--type spdxjson \
|
||||
--predicate sbom.spdx.json \
|
||||
--yes \
|
||||
${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
|
||||
|
||||
# ============================================================
|
||||
# Deploy Job - Pull image and restart on production server
|
||||
# ============================================================
|
||||
deploy:
|
||||
name: Deploy to Production
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Sync docker-compose.yml to server
|
||||
uses: appleboy/scp-action@v0.1.7
|
||||
with:
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ secrets.SSH_PORT || 22 }}
|
||||
source: "docker-compose.yml"
|
||||
target: ${{ secrets.DEPLOY_PATH }}
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '18'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Deploy via SSH
|
||||
uses: appleboy/ssh-action@v1.2.4
|
||||
with:
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ secrets.SSH_PORT || 22 }}
|
||||
envs: IMAGE
|
||||
script_stop: true
|
||||
script: |
|
||||
set -e
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
# Image to deploy (using specific SHA tag, not latest)
|
||||
IMAGE="${{ needs.build.outputs.full_image }}"
|
||||
echo "Deploying image: $IMAGE"
|
||||
- name: Build Docker image
|
||||
run: |
|
||||
docker build -t apartment-api:${{ github.sha }} .
|
||||
docker tag apartment-api:${{ github.sha }} apartment-api:latest
|
||||
|
||||
# Navigate to deployment directory
|
||||
cd ${{ secrets.DEPLOY_PATH }}
|
||||
- name: Deploy to server
|
||||
run: |
|
||||
# Copy files to deployment directory
|
||||
mkdir -p /media/stephen/Storage_Linux/infrastructure/services/apartment-api
|
||||
cp -r . /media/stephen/Storage_Linux/infrastructure/services/apartment-api/
|
||||
|
||||
# Pull the specific image (using pre-configured Docker credentials)
|
||||
docker pull "$IMAGE"
|
||||
# Navigate to deployment directory
|
||||
cd /media/stephen/Storage_Linux/infrastructure/services/apartment-api
|
||||
|
||||
# Update and restart with new image
|
||||
export IMAGE
|
||||
docker compose up -d
|
||||
# Stop existing container if running
|
||||
docker compose down || true
|
||||
|
||||
# Clean up old images
|
||||
docker image prune -f
|
||||
# Start new container
|
||||
docker compose up -d
|
||||
|
||||
# Wait for container to be healthy
|
||||
echo "Waiting for container to start..."
|
||||
sleep 10
|
||||
# Clean up old images
|
||||
docker image prune -f
|
||||
|
||||
# Verify container is running
|
||||
if docker compose ps | grep -q "Up"; then
|
||||
echo "Container is running successfully"
|
||||
else
|
||||
echo "ERROR: Container failed to start"
|
||||
docker compose logs --tail=50
|
||||
exit 1
|
||||
fi
|
||||
- name: Verify deployment
|
||||
run: |
|
||||
# Wait for container to start
|
||||
sleep 15
|
||||
|
||||
- name: Verify deployment
|
||||
uses: appleboy/ssh-action@v1.2.4
|
||||
with:
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ secrets.SSH_PORT || 22 }}
|
||||
script: |
|
||||
# Test health endpoint
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://apartments.maverickapplications.com/api/health || echo "000")
|
||||
# Check if container is running
|
||||
docker ps | grep apartment-api
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
echo "Health check passed (HTTP $HTTP_CODE)"
|
||||
else
|
||||
echo "WARNING: Health check returned HTTP $HTTP_CODE"
|
||||
echo "Container may still be initializing..."
|
||||
exit 1
|
||||
fi
|
||||
# Test health endpoint
|
||||
curl -f http://localhost:3000/health || echo "API may still be starting..."
|
||||
|
||||
# Test API endpoint
|
||||
curl -f http://localhost:3000/api/daily-summary || echo "API may still be loading data..."
|
||||
@ -1,12 +1,8 @@
|
||||
FROM node:20-alpine
|
||||
FROM node:18-alpine
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Update Alpine packages and npm to fix security vulnerabilities
|
||||
RUN apk upgrade --no-cache && \
|
||||
npm install -g npm@latest
|
||||
|
||||
# Install Curl
|
||||
RUN apk add --no-cache curl
|
||||
|
||||
@ -14,7 +10,7 @@ RUN apk add --no-cache curl
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm ci --omit=dev
|
||||
RUN npm ci --only=production
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
@ -361,13 +361,13 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.users).toBeDefined();
|
||||
expect(Array.isArray(response.body.data.users)).toBe(true);
|
||||
expect(response.body.data.pagination).toBeDefined();
|
||||
expect(response.body.data.pagination.page).toBe(1);
|
||||
expect(response.body.data.pagination.limit).toBe(20);
|
||||
expect(response.body.data.pagination.total).toBeGreaterThan(0);
|
||||
expect(response.body.data.pagination.pages).toBeDefined();
|
||||
expect(response.body.users).toBeDefined();
|
||||
expect(Array.isArray(response.body.users)).toBe(true);
|
||||
expect(response.body.pagination).toBeDefined();
|
||||
expect(response.body.pagination.page).toBe(1);
|
||||
expect(response.body.pagination.limit).toBe(20);
|
||||
expect(response.body.pagination.total).toBeGreaterThan(0);
|
||||
expect(response.body.pagination.pages).toBeDefined();
|
||||
});
|
||||
|
||||
it('should support page parameter', async () => {
|
||||
@ -383,7 +383,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.pagination.page).toBe(2);
|
||||
expect(response.body.pagination.page).toBe(2);
|
||||
});
|
||||
|
||||
it('should support limit parameter with max 100', async () => {
|
||||
@ -392,7 +392,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.pagination.limit).toBe(50);
|
||||
expect(response.body.pagination.limit).toBe(50);
|
||||
});
|
||||
|
||||
it('should cap limit at 100', async () => {
|
||||
@ -401,7 +401,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.pagination.limit).toBeLessThanOrEqual(100);
|
||||
expect(response.body.pagination.limit).toBeLessThanOrEqual(100);
|
||||
});
|
||||
|
||||
it('should support search parameter for name', async () => {
|
||||
@ -413,8 +413,8 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.users.some(u => u.name.includes('John'))).toBe(true);
|
||||
expect(response.body.data.users.every(u =>
|
||||
expect(response.body.users.some(u => u.name.includes('John'))).toBe(true);
|
||||
expect(response.body.users.every(u =>
|
||||
u.name.toLowerCase().includes('john') ||
|
||||
u.email.toLowerCase().includes('john')
|
||||
)).toBe(true);
|
||||
@ -428,7 +428,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.users.some(u => u.email.includes('unique-test'))).toBe(true);
|
||||
expect(response.body.users.some(u => u.email.includes('unique-test'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should support status filter "active"', async () => {
|
||||
@ -440,7 +440,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.users.every(u => u.isActive === true)).toBe(true);
|
||||
expect(response.body.users.every(u => u.isActive === true)).toBe(true);
|
||||
});
|
||||
|
||||
it('should support status filter "disabled"', async () => {
|
||||
@ -451,7 +451,7 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.users.every(u => u.isActive === false)).toBe(true);
|
||||
expect(response.body.users.every(u => u.isActive === false)).toBe(true);
|
||||
});
|
||||
|
||||
it('should support sort parameter', async () => {
|
||||
@ -472,7 +472,7 @@ describe('Phase 1: Foundation', () => {
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
// Newest first when sorted descending
|
||||
const createdDates = response.body.data.users.map(u => new Date(u.createdAt));
|
||||
const createdDates = response.body.users.map(u => new Date(u.createdAt));
|
||||
for (let i = 0; i < createdDates.length - 1; i++) {
|
||||
expect(createdDates[i] >= createdDates[i + 1]).toBe(true);
|
||||
}
|
||||
@ -485,7 +485,7 @@ describe('Phase 1: Foundation', () => {
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
// Verify ascending order
|
||||
const counts = response.body.data.users.map(u => u.loginCount);
|
||||
const counts = response.body.users.map(u => u.loginCount);
|
||||
for (let i = 0; i < counts.length - 1; i++) {
|
||||
expect(counts[i] <= counts[i + 1]).toBe(true);
|
||||
}
|
||||
@ -517,11 +517,11 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user).toBeDefined();
|
||||
expect(response.body.data.user._id.toString()).toBe(testUser._id.toString());
|
||||
expect(response.body.data.user.email).toBe(testUser.email);
|
||||
expect(response.body.data.user.name).toBe(testUser.name);
|
||||
expect(response.body.data.user.role).toBe(testUser.role);
|
||||
expect(response.body.user).toBeDefined();
|
||||
expect(response.body.user._id.toString()).toBe(testUser._id.toString());
|
||||
expect(response.body.user.email).toBe(testUser.email);
|
||||
expect(response.body.user.name).toBe(testUser.name);
|
||||
expect(response.body.user.role).toBe(testUser.role);
|
||||
});
|
||||
|
||||
it('should include recent activity for the user', async () => {
|
||||
@ -550,8 +550,8 @@ describe('Phase 1: Foundation', () => {
|
||||
.set('Cookie', [`auth_token=${adminToken}`]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user.recentActivity).toBeDefined();
|
||||
expect(Array.isArray(response.body.data.user.recentActivity)).toBe(true);
|
||||
expect(response.body.user.recentActivity).toBeDefined();
|
||||
expect(Array.isArray(response.body.user.recentActivity)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return 404 for non-existent user ID', async () => {
|
||||
@ -601,9 +601,9 @@ describe('Phase 1: Foundation', () => {
|
||||
.send({ isActive: false });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user).toBeDefined();
|
||||
expect(response.body.data.user.isActive).toBe(false);
|
||||
expect(response.body.data.message).toContain('disabled');
|
||||
expect(response.body.user).toBeDefined();
|
||||
expect(response.body.user.isActive).toBe(false);
|
||||
expect(response.body.message).toContain('disabled');
|
||||
|
||||
// Verify in database
|
||||
const dbUser = await db.collection('users').findOne({ _id: testUser._id });
|
||||
@ -620,9 +620,9 @@ describe('Phase 1: Foundation', () => {
|
||||
.send({ isActive: true });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user).toBeDefined();
|
||||
expect(response.body.data.user.isActive).toBe(true);
|
||||
expect(response.body.data.message).toContain('enabled');
|
||||
expect(response.body.user).toBeDefined();
|
||||
expect(response.body.user.isActive).toBe(true);
|
||||
expect(response.body.message).toContain('enabled');
|
||||
});
|
||||
|
||||
it('should set disabledAt timestamp when disabling', async () => {
|
||||
@ -729,9 +729,9 @@ describe('Phase 1: Foundation', () => {
|
||||
.send({ role: 'admin' });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user).toBeDefined();
|
||||
expect(response.body.data.user.role).toBe('admin');
|
||||
expect(response.body.data.message).toContain('promoted');
|
||||
expect(response.body.user).toBeDefined();
|
||||
expect(response.body.user.role).toBe('admin');
|
||||
expect(response.body.message).toContain('promoted');
|
||||
|
||||
// Verify in database
|
||||
const dbUser = await db.collection('users').findOne({ _id: testUser._id });
|
||||
@ -748,9 +748,9 @@ describe('Phase 1: Foundation', () => {
|
||||
.send({ role: 'user' });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.user).toBeDefined();
|
||||
expect(response.body.data.user.role).toBe('user');
|
||||
expect(response.body.data.message).toContain('demoted');
|
||||
expect(response.body.user).toBeDefined();
|
||||
expect(response.body.user.role).toBe('user');
|
||||
expect(response.body.message).toContain('demoted');
|
||||
});
|
||||
|
||||
it('should return 400 when trying to demote yourself from admin', async () => {
|
||||
@ -846,7 +846,7 @@ describe('Phase 1: Foundation', () => {
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
// Check that sensitive fields are not exposed
|
||||
const userInResponse = response.body.data.users.find(
|
||||
const userInResponse = response.body.users.find(
|
||||
u => u._id.toString() === testUser._id.toString()
|
||||
);
|
||||
|
||||
|
||||
@ -94,8 +94,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveProperty('activities');
|
||||
expect(res.body.data).toHaveProperty('pagination');
|
||||
expect(res.body).toHaveProperty('activities');
|
||||
expect(res.body).toHaveProperty('pagination');
|
||||
});
|
||||
});
|
||||
|
||||
@ -119,8 +119,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(50);
|
||||
expect(res.body.data.pagination).toEqual(expect.objectContaining({
|
||||
expect(res.body.activities).toHaveLength(50);
|
||||
expect(res.body.pagination).toEqual(expect.objectContaining({
|
||||
page: 1,
|
||||
limit: 50,
|
||||
total: 100,
|
||||
@ -146,8 +146,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(50);
|
||||
expect(res.body.data.pagination.page).toBe(2);
|
||||
expect(res.body.activities).toHaveLength(50);
|
||||
expect(res.body.pagination.page).toBe(2);
|
||||
});
|
||||
|
||||
it('should respect custom limit parameter', async () => {
|
||||
@ -168,9 +168,9 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(25);
|
||||
expect(res.body.data.pagination.limit).toBe(25);
|
||||
expect(res.body.data.pagination.pages).toBe(4);
|
||||
expect(res.body.activities).toHaveLength(25);
|
||||
expect(res.body.pagination.limit).toBe(25);
|
||||
expect(res.body.pagination.pages).toBe(4);
|
||||
});
|
||||
|
||||
it('should enforce maximum limit of 200', async () => {
|
||||
@ -183,7 +183,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.pagination.limit).toBeLessThanOrEqual(200);
|
||||
expect(res.body.pagination.limit).toBeLessThanOrEqual(200);
|
||||
});
|
||||
|
||||
it('should return empty array for page beyond available data', async () => {
|
||||
@ -204,7 +204,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(0);
|
||||
expect(res.body.activities).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@ -231,8 +231,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(20);
|
||||
res.body.data.activities.forEach(activity => {
|
||||
expect(res.body.activities).toHaveLength(20);
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity.userId.toString()).toBe(user1._id.toString());
|
||||
});
|
||||
});
|
||||
@ -256,8 +256,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(10);
|
||||
res.body.data.activities.forEach(activity => {
|
||||
expect(res.body.activities).toHaveLength(10);
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity.action).toBe('login');
|
||||
});
|
||||
});
|
||||
@ -286,7 +286,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
res.body.data.activities.forEach(activity => {
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(new Date(activity.timestamp).getTime()).toBeGreaterThanOrEqual(startDate.getTime());
|
||||
});
|
||||
});
|
||||
@ -315,7 +315,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
res.body.data.activities.forEach(activity => {
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(new Date(activity.timestamp).getTime()).toBeLessThanOrEqual(endDate.getTime());
|
||||
});
|
||||
});
|
||||
@ -340,7 +340,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
res.body.data.activities.forEach(activity => {
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity.userId.toString()).toBe(user1._id.toString());
|
||||
expect(activity.action).toBe('page_view');
|
||||
});
|
||||
@ -356,7 +356,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(0);
|
||||
expect(res.body.activities).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@ -377,7 +377,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
res.body.data.activities.forEach(activity => {
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity).toHaveProperty('userName');
|
||||
expect(activity.userName).toBe('Test User Name');
|
||||
});
|
||||
@ -399,7 +399,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
res.body.data.activities.forEach(activity => {
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity).toHaveProperty('userEmail');
|
||||
expect(activity.userEmail).toBe('testuser@example.com');
|
||||
});
|
||||
@ -421,9 +421,9 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities.length).toBeGreaterThan(0);
|
||||
expect(res.body.activities.length).toBeGreaterThan(0);
|
||||
// userName and userEmail should be null for deleted users
|
||||
const activity = res.body.data.activities[0];
|
||||
const activity = res.body.activities[0];
|
||||
expect(activity.userName).toBeNull();
|
||||
expect(activity.userEmail).toBeNull();
|
||||
});
|
||||
@ -450,7 +450,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
const timestamps = res.body.data.activities.map(a => new Date(a.timestamp).getTime());
|
||||
const timestamps = res.body.activities.map(a => new Date(a.timestamp).getTime());
|
||||
for (let i = 1; i < timestamps.length; i++) {
|
||||
expect(timestamps[i - 1]).toBeGreaterThanOrEqual(timestamps[i]);
|
||||
}
|
||||
@ -569,7 +569,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data).toHaveProperty('activities');
|
||||
expect(res.body).toHaveProperty('activities');
|
||||
});
|
||||
});
|
||||
|
||||
@ -596,8 +596,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(15);
|
||||
res.body.data.activities.forEach(activity => {
|
||||
expect(res.body.activities).toHaveLength(15);
|
||||
res.body.activities.forEach(activity => {
|
||||
expect(activity.userId.toString()).toBe(user1._id.toString());
|
||||
});
|
||||
});
|
||||
@ -647,8 +647,8 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(5);
|
||||
expect(res.body.data.pagination).toEqual(expect.objectContaining({
|
||||
expect(res.body.activities).toHaveLength(5);
|
||||
expect(res.body.pagination).toEqual(expect.objectContaining({
|
||||
page: 1,
|
||||
limit: 5,
|
||||
total: 15,
|
||||
@ -668,7 +668,7 @@ describe('Phase 2: Activity Monitoring API', () => {
|
||||
.set('Cookie', [`auth_token=${token}`])
|
||||
.expect(200);
|
||||
|
||||
expect(res.body.data.activities).toHaveLength(0);
|
||||
expect(res.body.activities).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -255,8 +255,8 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ activityRetentionDays: 60 })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toHaveProperty('settings');
|
||||
expect(response.body.data).toHaveProperty('message');
|
||||
expect(response.body).toHaveProperty('settings');
|
||||
expect(response.body).toHaveProperty('message');
|
||||
});
|
||||
});
|
||||
|
||||
@ -287,7 +287,7 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ activityRetentionDays: 30 })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.settings.activityRetentionDays).toBe(30);
|
||||
expect(response.body.settings.activityRetentionDays).toBe(30);
|
||||
});
|
||||
|
||||
it('should accept retention period at maximum boundary (365 days)', async () => {
|
||||
@ -296,7 +296,7 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ activityRetentionDays: 365 })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.settings.activityRetentionDays).toBe(365);
|
||||
expect(response.body.settings.activityRetentionDays).toBe(365);
|
||||
});
|
||||
|
||||
it('should accept valid retention period within range (180 days)', async () => {
|
||||
@ -305,7 +305,7 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ activityRetentionDays: 180 })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.settings.activityRetentionDays).toBe(180);
|
||||
expect(response.body.settings.activityRetentionDays).toBe(180);
|
||||
});
|
||||
|
||||
it('should return 400 for non-numeric retention period', async () => {
|
||||
@ -343,8 +343,8 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ activityRetentionDays: 120 })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.settings.activityRetentionDays).toBe(120);
|
||||
expect(response.body.data.message).toBe('Settings updated successfully');
|
||||
expect(response.body.settings.activityRetentionDays).toBe(120);
|
||||
expect(response.body.message).toBe('Settings updated successfully');
|
||||
});
|
||||
|
||||
it('should persist the updated settings in database', async () => {
|
||||
@ -643,7 +643,7 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ role: 'user' })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.user.role).toBe('user');
|
||||
expect(response.body.user.role).toBe('user');
|
||||
});
|
||||
|
||||
it('should not allow self-demotion even when other admins exist', async () => {
|
||||
@ -688,7 +688,7 @@ describe('Phase 4: Settings & Security', () => {
|
||||
.send({ role: 'user' })
|
||||
.expect(200);
|
||||
|
||||
expect(response1.body.data.user.role).toBe('user');
|
||||
expect(response1.body.user.role).toBe('user');
|
||||
|
||||
// Now the primary admin is the last one, cannot demote
|
||||
const response2 = await authAs(testUsers.admin)
|
||||
@ -702,9 +702,8 @@ describe('Phase 4: Settings & Security', () => {
|
||||
|
||||
// ============================================================
|
||||
// SEC-4.3: Rate Limiting on Admin Endpoints
|
||||
// SKIPPED: Rate limiting moved to Phase 5
|
||||
// ============================================================
|
||||
describe.skip('SEC-4.3: Rate Limiting', () => {
|
||||
describe('SEC-4.3: Rate Limiting', () => {
|
||||
/**
|
||||
* Helper to make multiple rapid requests
|
||||
* @param {string} endpoint - API endpoint
|
||||
@ -918,8 +917,8 @@ describe('Integration: Settings Update Workflow', () => {
|
||||
.send({ activityRetentionDays: 180 })
|
||||
.expect(200);
|
||||
|
||||
expect(updateResponse.body.data.settings.activityRetentionDays).toBe(180);
|
||||
expect(updateResponse.body.data.message).toBe('Settings updated successfully');
|
||||
expect(updateResponse.body.settings.activityRetentionDays).toBe(180);
|
||||
expect(updateResponse.body.message).toBe('Settings updated successfully');
|
||||
|
||||
// Step 3: Verify settings persisted
|
||||
const verifyResponse = await authAdmin('get', '/api/admin/settings')
|
||||
@ -963,7 +962,7 @@ describe('Integration: Settings Update Workflow', () => {
|
||||
.send({ activityRetentionDays: value })
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data.settings.activityRetentionDays).toBe(value);
|
||||
expect(response.body.settings.activityRetentionDays).toBe(value);
|
||||
|
||||
// Verify TTL index after each update
|
||||
const indexes = await db.collection('user_activity').indexes();
|
||||
|
||||
@ -1,496 +0,0 @@
|
||||
/**
|
||||
* Tests for convertDataTypes() and its internal helper functions
|
||||
*
|
||||
* Covers:
|
||||
* - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
|
||||
* - parseInteger: valid integers, null/empty/invalid
|
||||
* - parsePositiveInteger: treats 0 as null (for area)
|
||||
* - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
|
||||
* - parseFloat: "1.5" for half baths, null/empty/invalid
|
||||
* - parseBoolean: "true"/"false"/"1"/"0", null/empty
|
||||
* - trimString: trim whitespace, null for empty, null/undefined
|
||||
* - convertDataTypes: full integration applying correct parser to each field
|
||||
*/
|
||||
|
||||
const {
|
||||
convertDataTypes,
|
||||
parseInteger,
|
||||
parsePositiveInteger,
|
||||
parseIntegerOrString,
|
||||
parsePrice,
|
||||
parseBoolean,
|
||||
parseFloatValue,
|
||||
trimString
|
||||
} = require('../../services/scraperService');
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. parsePrice
|
||||
// ---------------------------------------------------------------
|
||||
describe('parsePrice', () => {
|
||||
it('should parse "$1,234" format to 1234', () => {
|
||||
expect(parsePrice('$1,234')).toBe(1234);
|
||||
});
|
||||
|
||||
it('should parse plain numeric string', () => {
|
||||
expect(parsePrice('1450')).toBe(1450);
|
||||
});
|
||||
|
||||
it('should parse "$2,500.00" stripping non-numeric chars', () => {
|
||||
expect(parsePrice('$2,500.00')).toBe(2500);
|
||||
});
|
||||
|
||||
it('should return null for "Call for pricing"', () => {
|
||||
expect(parsePrice('Call for pricing')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for "Contact us"', () => {
|
||||
expect(parsePrice('Contact us')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for "Please inquire"', () => {
|
||||
expect(parsePrice('Please inquire')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for negative values', () => {
|
||||
// After stripping non-numeric chars, "-1500" becomes "1500"
|
||||
// but if someone passes a numeric -1500 directly, check that too
|
||||
expect(parsePrice('-')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parsePrice(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parsePrice('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parsePrice(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for completely non-numeric string', () => {
|
||||
expect(parsePrice('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle price of 0', () => {
|
||||
expect(parsePrice('0')).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle large prices', () => {
|
||||
expect(parsePrice('$15,500')).toBe(15500);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. parseInteger
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseInteger', () => {
|
||||
it('should parse valid integer string', () => {
|
||||
expect(parseInteger('42')).toBe(42);
|
||||
});
|
||||
|
||||
it('should parse "0" to 0', () => {
|
||||
expect(parseInteger('0')).toBe(0);
|
||||
});
|
||||
|
||||
it('should parse negative integer string', () => {
|
||||
expect(parseInteger('-5')).toBe(-5);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseInteger(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseInteger('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseInteger(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parseInteger('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for NaN', () => {
|
||||
expect(parseInteger('NaN')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for Infinity', () => {
|
||||
expect(parseInteger('Infinity')).toBeNull();
|
||||
});
|
||||
|
||||
it('should truncate float strings to integer', () => {
|
||||
expect(parseInteger('3.7')).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. parsePositiveInteger
|
||||
// ---------------------------------------------------------------
|
||||
describe('parsePositiveInteger', () => {
|
||||
it('should parse positive integer normally', () => {
|
||||
expect(parsePositiveInteger('750')).toBe(750);
|
||||
});
|
||||
|
||||
it('should treat 0 as null (for area field)', () => {
|
||||
expect(parsePositiveInteger('0')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null input', () => {
|
||||
expect(parsePositiveInteger(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parsePositiveInteger('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parsePositiveInteger('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle negative values (parsed as negative int, not positive)', () => {
|
||||
expect(parsePositiveInteger('-5')).toBe(-5);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. parseIntegerOrString
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseIntegerOrString', () => {
|
||||
it('should return integer when value is cleanly parseable', () => {
|
||||
expect(parseIntegerOrString('5001')).toBe(5001);
|
||||
});
|
||||
|
||||
it('should return trimmed string for non-numeric value', () => {
|
||||
expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
|
||||
});
|
||||
|
||||
it('should return trimmed string when value has non-numeric suffix', () => {
|
||||
expect(parseIntegerOrString('123abc')).toBe('123abc');
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseIntegerOrString(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseIntegerOrString('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseIntegerOrString(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should trim whitespace before checking', () => {
|
||||
expect(parseIntegerOrString(' 5001 ')).toBe(5001);
|
||||
});
|
||||
|
||||
it('should return trimmed string for non-numeric with whitespace', () => {
|
||||
expect(parseIntegerOrString(' abc ')).toBe('abc');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. parseFloatValue
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseFloatValue', () => {
|
||||
it('should parse "1.5" for half baths', () => {
|
||||
expect(parseFloatValue('1.5')).toBe(1.5);
|
||||
});
|
||||
|
||||
it('should parse "2.0" to 2', () => {
|
||||
expect(parseFloatValue('2.0')).toBe(2.0);
|
||||
});
|
||||
|
||||
it('should parse integer string as float', () => {
|
||||
expect(parseFloatValue('3')).toBe(3);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseFloatValue(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseFloatValue('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseFloatValue(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parseFloatValue('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for NaN', () => {
|
||||
expect(parseFloatValue('NaN')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for Infinity', () => {
|
||||
expect(parseFloatValue('Infinity')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. parseBoolean
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseBoolean', () => {
|
||||
it('should parse "true" to true', () => {
|
||||
expect(parseBoolean('true')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "false" to false', () => {
|
||||
expect(parseBoolean('false')).toBe(false);
|
||||
});
|
||||
|
||||
it('should parse "1" to true', () => {
|
||||
expect(parseBoolean('1')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "0" to false', () => {
|
||||
expect(parseBoolean('0')).toBe(false);
|
||||
});
|
||||
|
||||
it('should parse "TRUE" (case insensitive) to true', () => {
|
||||
expect(parseBoolean('TRUE')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "False" (case insensitive) to false', () => {
|
||||
expect(parseBoolean('False')).toBe(false);
|
||||
});
|
||||
|
||||
it('should pass through boolean true', () => {
|
||||
expect(parseBoolean(true)).toBe(true);
|
||||
});
|
||||
|
||||
it('should pass through boolean false', () => {
|
||||
expect(parseBoolean(false)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseBoolean(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseBoolean('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseBoolean(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for unrecognized string', () => {
|
||||
expect(parseBoolean('yes')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. trimString
|
||||
// ---------------------------------------------------------------
|
||||
describe('trimString', () => {
|
||||
it('should trim leading and trailing whitespace', () => {
|
||||
expect(trimString(' hello ')).toBe('hello');
|
||||
});
|
||||
|
||||
it('should return the string as-is if no whitespace', () => {
|
||||
expect(trimString('hello')).toBe('hello');
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(trimString('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for whitespace-only string', () => {
|
||||
expect(trimString(' ')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(trimString(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(trimString(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should convert non-string to string then trim', () => {
|
||||
expect(trimString(123)).toBe('123');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. convertDataTypes - Full integration test
|
||||
// ---------------------------------------------------------------
|
||||
describe('convertDataTypes', () => {
|
||||
it('should apply correct parser to each field', () => {
|
||||
const rawUnit = {
|
||||
id: '1001',
|
||||
unit_code: 'CCT-101',
|
||||
unit_id: '5001',
|
||||
floor: '1',
|
||||
area: '750',
|
||||
bed_count: '1',
|
||||
bath_count: '1.5',
|
||||
price: '1450',
|
||||
available: 'true',
|
||||
unavailable: 'false',
|
||||
soonest: '2026-03-01',
|
||||
date_available: '20260301',
|
||||
plan_id: '201',
|
||||
plan_name: 'Alpine',
|
||||
obj_type: 'unit',
|
||||
community: 'Country Club Towers',
|
||||
asset: '100',
|
||||
href: '/units/CCT-101',
|
||||
inventory_href: '/inventory/CCT-101',
|
||||
image_url: 'https://example.com/images/unit-101.jpg',
|
||||
specials_content: 'First month free!'
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
// Integer fields
|
||||
expect(converted.id).toBe(1001);
|
||||
expect(converted.floor).toBe(1);
|
||||
expect(converted.bed_count).toBe(1);
|
||||
expect(converted.plan_id).toBe(201);
|
||||
expect(converted.asset).toBe(100);
|
||||
expect(converted.date_available).toBe(20260301);
|
||||
|
||||
// parseIntegerOrString
|
||||
expect(converted.unit_id).toBe(5001);
|
||||
|
||||
// parsePositiveInteger (area)
|
||||
expect(converted.area).toBe(750);
|
||||
|
||||
// Float field (bath_count)
|
||||
expect(converted.bath_count).toBe(1.5);
|
||||
|
||||
// Price field
|
||||
expect(converted.price).toBe(1450);
|
||||
|
||||
// Boolean fields
|
||||
expect(converted.available).toBe(true);
|
||||
expect(converted.unavailable).toBe(false);
|
||||
|
||||
// String fields (trimmed)
|
||||
expect(converted.unit_code).toBe('CCT-101');
|
||||
expect(converted.plan_name).toBe('Alpine');
|
||||
expect(converted.soonest).toBe('2026-03-01');
|
||||
expect(converted.obj_type).toBe('unit');
|
||||
expect(converted.community).toBe('Country Club Towers');
|
||||
expect(converted.href).toBe('/units/CCT-101');
|
||||
expect(converted.inventory_href).toBe('/inventory/CCT-101');
|
||||
expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
|
||||
expect(converted.specials_content).toBe('First month free!');
|
||||
});
|
||||
|
||||
it('should preserve string fields without converting them', () => {
|
||||
const rawUnit = {
|
||||
unit_code: 'CCT-205',
|
||||
plan_name: ' Birch ',
|
||||
soonest: 'Now',
|
||||
obj_type: 'unit',
|
||||
community: ' Country Club Towers ',
|
||||
href: '/units/CCT-205',
|
||||
inventory_href: '/inventory/CCT-205',
|
||||
image_url: 'https://example.com/img.jpg',
|
||||
specials_content: null
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
expect(converted.unit_code).toBe('CCT-205');
|
||||
expect(converted.plan_name).toBe('Birch');
|
||||
expect(converted.community).toBe('Country Club Towers');
|
||||
expect(converted.specials_content).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle a unit with all null/undefined/empty values', () => {
|
||||
const rawUnit = {
|
||||
id: null,
|
||||
unit_code: undefined,
|
||||
unit_id: '',
|
||||
floor: null,
|
||||
area: '',
|
||||
bed_count: undefined,
|
||||
bath_count: null,
|
||||
price: '',
|
||||
available: null,
|
||||
unavailable: undefined,
|
||||
soonest: '',
|
||||
date_available: null,
|
||||
plan_id: undefined,
|
||||
plan_name: '',
|
||||
obj_type: null,
|
||||
community: undefined,
|
||||
asset: '',
|
||||
href: null,
|
||||
inventory_href: undefined,
|
||||
image_url: '',
|
||||
specials_content: null
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
expect(converted.id).toBeNull();
|
||||
expect(converted.unit_code).toBeNull();
|
||||
expect(converted.unit_id).toBeNull();
|
||||
expect(converted.floor).toBeNull();
|
||||
expect(converted.area).toBeNull();
|
||||
expect(converted.bed_count).toBeNull();
|
||||
expect(converted.bath_count).toBeNull();
|
||||
expect(converted.price).toBeNull();
|
||||
expect(converted.available).toBeNull();
|
||||
expect(converted.unavailable).toBeNull();
|
||||
expect(converted.soonest).toBeNull();
|
||||
expect(converted.date_available).toBeNull();
|
||||
expect(converted.plan_id).toBeNull();
|
||||
expect(converted.plan_name).toBeNull();
|
||||
expect(converted.obj_type).toBeNull();
|
||||
expect(converted.community).toBeNull();
|
||||
expect(converted.asset).toBeNull();
|
||||
expect(converted.href).toBeNull();
|
||||
expect(converted.inventory_href).toBeNull();
|
||||
expect(converted.image_url).toBeNull();
|
||||
expect(converted.specials_content).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle area=0 as null (parsePositiveInteger)', () => {
|
||||
const rawUnit = { area: '0' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.area).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle bed_count=0 as valid (studio)', () => {
|
||||
const rawUnit = { bed_count: '0' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.bed_count).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle "Call for pricing" in price field', () => {
|
||||
const rawUnit = { price: 'Call for pricing' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.price).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle "$1,234" price format', () => {
|
||||
const rawUnit = { price: '$1,234' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.price).toBe(1234);
|
||||
});
|
||||
|
||||
it('should handle unit_id as string identifier', () => {
|
||||
const rawUnit = { unit_id: 'UNIT-ABC' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.unit_id).toBe('UNIT-ABC');
|
||||
});
|
||||
|
||||
it('should handle bath_count with half baths', () => {
|
||||
const rawUnit = { bath_count: '1.5' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.bath_count).toBe(1.5);
|
||||
});
|
||||
});
|
||||
@ -1,535 +0,0 @@
|
||||
/**
|
||||
* Tests for fetchPage function
|
||||
*
|
||||
* Acceptance Criteria:
|
||||
* - Uses axios for HTTP GET
|
||||
* - Enforces timeout from config (default 30s)
|
||||
* - Sets User-Agent header from config
|
||||
* - Implements retry with exponential backoff (1s, 2s, 4s)
|
||||
* - Retries on 5xx errors and network timeouts
|
||||
* - Does NOT retry on 4xx errors
|
||||
* - Logs each attempt with attempt number
|
||||
* - Returns HTML string on success
|
||||
* - Throws error after all retries exhausted
|
||||
*/
|
||||
|
||||
const axios = require('axios');
|
||||
|
||||
// Mock axios
|
||||
jest.mock('axios');
|
||||
|
||||
// Mock config
|
||||
jest.mock('../../config/scraper', () => ({
|
||||
RETRY_CONFIG: {
|
||||
maxRetries: 3,
|
||||
baseDelay: 1000,
|
||||
timeout: 30000
|
||||
},
|
||||
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
|
||||
}));
|
||||
|
||||
// Import after mocking
|
||||
const { fetchPage } = require('../../services/scraperService');
|
||||
const config = require('../../config/scraper');
|
||||
|
||||
describe('fetchPage', () => {
|
||||
let mockLogger;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
jest.useFakeTimers();
|
||||
|
||||
// Create mock logger
|
||||
mockLogger = {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
describe('successful requests', () => {
|
||||
it('should return HTML string on success', async () => {
|
||||
const htmlContent = '<html><body>Test content</body></html>';
|
||||
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: htmlContent
|
||||
});
|
||||
|
||||
const result = await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(result).toBe(htmlContent);
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should use axios for HTTP GET', async () => {
|
||||
const url = 'https://example.com/apartments';
|
||||
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage(url, mockLogger);
|
||||
|
||||
expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
|
||||
});
|
||||
|
||||
it('should set timeout from config', async () => {
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(axios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
timeout: config.RETRY_CONFIG.timeout
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should set User-Agent header from config', async () => {
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(axios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'User-Agent': config.USER_AGENT
|
||||
})
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log the attempt with attempt number', async () => {
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Fetching page',
|
||||
expect.objectContaining({
|
||||
url: 'https://example.com',
|
||||
attempt: 1
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log successful fetch with status and content length', async () => {
|
||||
const htmlContent = '<html><body>Content</body></html>';
|
||||
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: htmlContent
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Page fetched successfully',
|
||||
expect.objectContaining({
|
||||
status: 200,
|
||||
contentLength: htmlContent.length
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry behavior on 5xx errors', () => {
|
||||
it('should retry on 500 Internal Server Error', async () => {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
|
||||
// Fast-forward through the retry delay
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
const result = await promise;
|
||||
|
||||
expect(result).toBe('<html></html>');
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on 502 Bad Gateway', async () => {
|
||||
const error502 = new Error('Bad Gateway');
|
||||
error502.response = { status: 502 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error502)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on 503 Service Unavailable', async () => {
|
||||
const error503 = new Error('Service Unavailable');
|
||||
error503.response = { status: 503 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error503)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on 504 Gateway Timeout', async () => {
|
||||
const error504 = new Error('Gateway Timeout');
|
||||
error504.response = { status: 504 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error504)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry behavior on network errors', () => {
|
||||
it('should retry on network timeout (ECONNABORTED)', async () => {
|
||||
const timeoutError = new Error('timeout of 30000ms exceeded');
|
||||
timeoutError.code = 'ECONNABORTED';
|
||||
// Network errors don't have a response property
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(timeoutError)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
const result = await promise;
|
||||
|
||||
expect(result).toBe('<html></html>');
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
|
||||
const dnsError = new Error('getaddrinfo ENOTFOUND');
|
||||
dnsError.code = 'ENOTFOUND';
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(dnsError)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on connection refused (ECONNREFUSED)', async () => {
|
||||
const connError = new Error('connect ECONNREFUSED');
|
||||
connError.code = 'ECONNREFUSED';
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(connError)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('no retry on 4xx errors', () => {
|
||||
it('should NOT retry on 400 Bad Request', async () => {
|
||||
const error400 = new Error('Bad Request');
|
||||
error400.response = { status: 400 };
|
||||
|
||||
axios.get.mockRejectedValueOnce(error400);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow();
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should NOT retry on 401 Unauthorized', async () => {
|
||||
const error401 = new Error('Unauthorized');
|
||||
error401.response = { status: 401 };
|
||||
|
||||
axios.get.mockRejectedValueOnce(error401);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow();
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should NOT retry on 403 Forbidden', async () => {
|
||||
const error403 = new Error('Forbidden');
|
||||
error403.response = { status: 403 };
|
||||
|
||||
axios.get.mockRejectedValueOnce(error403);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow();
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should NOT retry on 404 Not Found', async () => {
|
||||
const error404 = new Error('Not Found');
|
||||
error404.response = { status: 404 };
|
||||
|
||||
axios.get.mockRejectedValueOnce(error404);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow();
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should NOT retry on 429 Too Many Requests', async () => {
|
||||
const error429 = new Error('Too Many Requests');
|
||||
error429.response = { status: 429 };
|
||||
|
||||
axios.get.mockRejectedValueOnce(error429);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow();
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('exponential backoff', () => {
|
||||
it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
// Fail 3 times, then succeed
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
|
||||
// First retry: 1 second delay
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
// Second retry: 2 second delay
|
||||
await jest.advanceTimersByTimeAsync(2000);
|
||||
|
||||
// Third retry: 4 second delay
|
||||
await jest.advanceTimersByTimeAsync(4000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('should log each retry attempt with attempt number', async () => {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
await jest.advanceTimersByTimeAsync(2000);
|
||||
|
||||
await promise;
|
||||
|
||||
// Check that attempt numbers were logged
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Fetching page',
|
||||
expect.objectContaining({ attempt: 1 })
|
||||
);
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Fetching page',
|
||||
expect.objectContaining({ attempt: 2 })
|
||||
);
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Fetching page',
|
||||
expect.objectContaining({ attempt: 3 })
|
||||
);
|
||||
});
|
||||
|
||||
it('should log failed attempts with error details', async () => {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(
|
||||
'Fetch attempt failed',
|
||||
expect.objectContaining({
|
||||
attempt: 1,
|
||||
statusCode: 500,
|
||||
isRetryable: true
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log wait time before retry', async () => {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockResolvedValueOnce({ status: 200, data: '<html></html>' });
|
||||
|
||||
const promise = fetchPage('https://example.com', mockLogger);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Waiting before retry',
|
||||
expect.objectContaining({ delay: 1000 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry exhaustion', () => {
|
||||
// Use real timers for exhaustion tests to avoid promise handling issues
|
||||
beforeEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
it('should throw error after all retries exhausted (4 attempts total)', async () => {
|
||||
// Override config for faster tests
|
||||
const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
|
||||
config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
|
||||
|
||||
try {
|
||||
const error500 = new Error('Internal Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
// All 4 attempts fail (1 initial + 3 retries)
|
||||
axios.get
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500)
|
||||
.mockRejectedValueOnce(error500);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toThrow('Internal Server Error');
|
||||
|
||||
expect(axios.get).toHaveBeenCalledTimes(4);
|
||||
} finally {
|
||||
config.RETRY_CONFIG.baseDelay = originalBaseDelay;
|
||||
}
|
||||
});
|
||||
|
||||
it('should throw the last error when retries are exhausted', async () => {
|
||||
// Override config for faster tests
|
||||
const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
|
||||
config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
|
||||
|
||||
try {
|
||||
const error500 = new Error('Server Error');
|
||||
error500.response = { status: 500 };
|
||||
|
||||
axios.get.mockRejectedValue(error500);
|
||||
|
||||
await expect(fetchPage('https://example.com', mockLogger))
|
||||
.rejects.toBe(error500);
|
||||
} finally {
|
||||
config.RETRY_CONFIG.baseDelay = originalBaseDelay;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('axios configuration', () => {
|
||||
it('should set maxRedirects to 5', async () => {
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
expect(axios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
maxRedirects: 5
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should accept 2xx and 3xx status codes via validateStatus', async () => {
|
||||
axios.get.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: '<html></html>'
|
||||
});
|
||||
|
||||
await fetchPage('https://example.com', mockLogger);
|
||||
|
||||
const callArgs = axios.get.mock.calls[0][1];
|
||||
const validateStatus = callArgs.validateStatus;
|
||||
|
||||
// Should accept 2xx
|
||||
expect(validateStatus(200)).toBe(true);
|
||||
expect(validateStatus(201)).toBe(true);
|
||||
expect(validateStatus(204)).toBe(true);
|
||||
|
||||
// Should accept 3xx
|
||||
expect(validateStatus(301)).toBe(true);
|
||||
expect(validateStatus(302)).toBe(true);
|
||||
|
||||
// Should reject 4xx and 5xx
|
||||
expect(validateStatus(400)).toBe(false);
|
||||
expect(validateStatus(404)).toBe(false);
|
||||
expect(validateStatus(500)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,64 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>Country Club Towers - Call for Pricing</title></head>
|
||||
<body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article
|
||||
class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
|
||||
data-spaces-id="9001"
|
||||
data-spaces-soonest="Now"
|
||||
data-spaces-sort-date="1706745600"
|
||||
data-spaces-sort-price="Call for pricing"
|
||||
data-spaces-sort-area="750"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="P-101"
|
||||
data-spaces-unit-id="9001"
|
||||
data-spaces-unit-floor="1"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="500"
|
||||
data-spaces-sort-plan-name="Penthouse A"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=9001"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
|
||||
aria-label="Unit P-101">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">Call for pricing</span>
|
||||
<span class="spaces-unit__unit">P-101</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
|
||||
data-spaces-id="9002"
|
||||
data-spaces-soonest="Now"
|
||||
data-spaces-sort-date="1706745600"
|
||||
data-spaces-sort-price="Call for pricing"
|
||||
data-spaces-sort-area="900"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-unit="P-102"
|
||||
data-spaces-unit-id="9002"
|
||||
data-spaces-unit-floor="2"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="501"
|
||||
data-spaces-sort-plan-name="Penthouse B"
|
||||
data-spaces-sort-bath="2"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=9002"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
|
||||
aria-label="Unit P-102">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">Call for pricing</span>
|
||||
<span class="spaces-unit__unit">P-102</span>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,9 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>Country Club Towers - No Units</title></head>
|
||||
<body>
|
||||
<section class="spaces__tab-unit">
|
||||
<!-- Empty - no units available -->
|
||||
</section>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,288 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>Country Club Towers - Apartments</title></head>
|
||||
<body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article
|
||||
class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154842"
|
||||
data-spaces-soonest="2025-10-11"
|
||||
data-spaces-sort-date="1760140800"
|
||||
data-spaces-sort-price="5230"
|
||||
data-spaces-sort-area="1210"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-unit="W2707"
|
||||
data-spaces-unit-id="154842"
|
||||
data-spaces-unit-floor="2755"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="10270"
|
||||
data-spaces-sort-plan-name="Pyramid Peak - Terrace"
|
||||
data-spaces-sort-bath="2"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154842"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
|
||||
aria-label="Unit W2707">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">5230</span>
|
||||
<span class="spaces-unit__unit">W2707</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154451"
|
||||
data-spaces-soonest="2026-03-08"
|
||||
data-spaces-sort-date="1772928000"
|
||||
data-spaces-sort-price="2767"
|
||||
data-spaces-sort-area="806"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="E0901"
|
||||
data-spaces-unit-id="154451"
|
||||
data-spaces-unit-floor="2738"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8016"
|
||||
data-spaces-sort-plan-name="Sunshine Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154451"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
|
||||
aria-label="Unit E0901">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2767</span>
|
||||
<span class="spaces-unit__unit">E0901</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154814"
|
||||
data-spaces-soonest="2025-12-18"
|
||||
data-spaces-sort-date="1766016000"
|
||||
data-spaces-sort-price="2255"
|
||||
data-spaces-sort-area="623"
|
||||
data-spaces-sort-bed="0"
|
||||
data-spaces-unit="W2603"
|
||||
data-spaces-unit-id="154814"
|
||||
data-spaces-unit-floor="2754"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8018"
|
||||
data-spaces-sort-plan-name="Little Bear Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154814"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
|
||||
aria-label="Unit W2603">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2255</span>
|
||||
<span class="spaces-unit__unit">W2603</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154428"
|
||||
data-spaces-soonest="2026-02-22"
|
||||
data-spaces-sort-date="1771718400"
|
||||
data-spaces-sort-price="4250"
|
||||
data-spaces-sort-area="1152"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-unit="W0802"
|
||||
data-spaces-unit-id="154428"
|
||||
data-spaces-unit-floor="2737"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8017"
|
||||
data-spaces-sort-plan-name="Mt. Princeton"
|
||||
data-spaces-sort-bath="2"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154428"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
|
||||
aria-label="Unit W0802">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">4250</span>
|
||||
<span class="spaces-unit__unit">W0802</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154778"
|
||||
data-spaces-soonest="2025-12-31"
|
||||
data-spaces-sort-date="1767139200"
|
||||
data-spaces-sort-price="2495"
|
||||
data-spaces-sort-area="764"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="W2405"
|
||||
data-spaces-unit-id="154778"
|
||||
data-spaces-unit-floor="2752"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8020"
|
||||
data-spaces-sort-plan-name="Grays Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154778"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
|
||||
aria-label="Unit W2405">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2495</span>
|
||||
<span class="spaces-unit__unit">W2405</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154442"
|
||||
data-spaces-soonest="2025-02-01"
|
||||
data-spaces-sort-date="1738368000"
|
||||
data-spaces-sort-price="2495"
|
||||
data-spaces-sort-area="802"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="W0809"
|
||||
data-spaces-unit-id="154442"
|
||||
data-spaces-unit-floor="2737"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8022"
|
||||
data-spaces-sort-plan-name="Pikes Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154442"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
|
||||
aria-label="Unit W0809">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2495</span>
|
||||
<span class="spaces-unit__unit">W0809</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154764"
|
||||
data-spaces-soonest="2026-02-17"
|
||||
data-spaces-sort-date="1771286400"
|
||||
data-spaces-sort-price="2435"
|
||||
data-spaces-sort-area="715"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="W2308"
|
||||
data-spaces-unit-id="154764"
|
||||
data-spaces-unit-floor="2751"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8023"
|
||||
data-spaces-sort-plan-name="Longs Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154764"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
|
||||
aria-label="Unit W2308">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2435</span>
|
||||
<span class="spaces-unit__unit">W2308</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154612"
|
||||
data-spaces-soonest="2026-03-24"
|
||||
data-spaces-sort-date="1774310400"
|
||||
data-spaces-sort-price="2683"
|
||||
data-spaces-sort-area="797"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-unit="W1610"
|
||||
data-spaces-unit-id="154612"
|
||||
data-spaces-unit-floor="2744"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8024"
|
||||
data-spaces-sort-plan-name="Torreys Peak"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154612"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
|
||||
aria-label="Unit W1610">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">2683</span>
|
||||
<span class="spaces-unit__unit">W1610</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
|
||||
data-spaces-id="154495"
|
||||
data-spaces-soonest="2025-11-07"
|
||||
data-spaces-sort-date="1762473600"
|
||||
data-spaces-sort-price="4470"
|
||||
data-spaces-sort-area="1230"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-unit="E1011"
|
||||
data-spaces-unit-id="154495"
|
||||
data-spaces-unit-floor="2739"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8025"
|
||||
data-spaces-sort-plan-name="Maroon Peak"
|
||||
data-spaces-sort-bath="2"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154495"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
|
||||
aria-label="Unit E1011">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">4470</span>
|
||||
<span class="spaces-unit__unit">E1011</span>
|
||||
</div>
|
||||
</article>
|
||||
<article
|
||||
class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
|
||||
data-spaces-id="154919"
|
||||
data-spaces-soonest="2025-10-27"
|
||||
data-spaces-sort-date="1761523200"
|
||||
data-spaces-sort-price="8581"
|
||||
data-spaces-sort-area="1532"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-unit="E3205"
|
||||
data-spaces-unit-id="154919"
|
||||
data-spaces-unit-floor="2760"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-available="true"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="420"
|
||||
data-spaces-specials-content=""
|
||||
data-spaces-plan-id="8037"
|
||||
data-spaces-sort-plan-name="Mt. Antero"
|
||||
data-spaces-sort-bath="2.5"
|
||||
data-spaces-href="?spaces_tab=unit-detail&detail=154919"
|
||||
data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
|
||||
aria-label="Unit E3205">
|
||||
<div class="spaces-unit__inner">
|
||||
<span class="spaces-unit__price">8581</span>
|
||||
<span class="spaces-unit__unit">E3205</span>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,583 +0,0 @@
|
||||
/**
|
||||
* Tests for insertPrices() bulk operation
|
||||
*
|
||||
* Covers:
|
||||
* - Only units with non-null prices are inserted
|
||||
* - bulkWrite uses updateOne with upsert for idempotency
|
||||
* - Filter uses unit_code + date_checked composite key
|
||||
* - Price record includes unit_code, date_checked, price, last_updated, data_source
|
||||
* - Empty price records logs warning and returns early
|
||||
* - Error handling when bulkWrite fails
|
||||
* - Logging of inserted and updated counts
|
||||
* - Re-running on same day updates existing records (idempotent)
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require insertPrices and getNow after implementation
|
||||
let insertPrices;
|
||||
let getNow;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
const PRICES_COLLECTION = 'unit_prices_scraper';
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ insertPrices, getNow } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean the prices collection before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('insertPrices', () => {
|
||||
const TODAY = '2026-02-05';
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. Only units with non-null prices are inserted
|
||||
// ---------------------------------------------------------------
|
||||
describe('filtering units with null prices', () => {
|
||||
it('should only insert records for units with non-null prices', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'A101', price: 1200 },
|
||||
{ unit_code: 'A102', price: null },
|
||||
{ unit_code: 'A103', price: 1500 },
|
||||
{ unit_code: 'A104', price: null }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(2);
|
||||
|
||||
const unitCodes = docs.map(d => d.unit_code).sort();
|
||||
expect(unitCodes).toEqual(['A101', 'A103']);
|
||||
});
|
||||
|
||||
it('should skip units with price of null and not create any record for them', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'B101', price: null },
|
||||
{ unit_code: 'B102', price: null }
|
||||
];
|
||||
|
||||
const result = await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
expect(result.insertedCount).toBe(0);
|
||||
|
||||
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle a mix of priced and null-priced units', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'C101', price: 950 },
|
||||
{ unit_code: 'C102', price: null },
|
||||
{ unit_code: 'C103', price: 1100 },
|
||||
{ unit_code: 'C104', price: null },
|
||||
{ unit_code: 'C105', price: 1300 }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const count = await db.collection(PRICES_COLLECTION).countDocuments();
|
||||
expect(count).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. bulkWrite uses updateOne with upsert for idempotency
|
||||
// ---------------------------------------------------------------
|
||||
describe('bulkWrite with upsert operations', () => {
|
||||
it('should create updateOne operations for each priced unit', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'D101', price: 1200 },
|
||||
{ unit_code: 'D102', price: 1500 }
|
||||
];
|
||||
|
||||
const result = await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// Both units should be upserted (new inserts)
|
||||
expect(result.insertedCount).toBeGreaterThan(0);
|
||||
|
||||
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should use upsert so re-running does not create duplicates', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'D201', price: 1200 }];
|
||||
|
||||
// First run
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// Second run on same day
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// Should still have only 1 document (not 2)
|
||||
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Filter uses unit_code + date_checked composite key
|
||||
// ---------------------------------------------------------------
|
||||
describe('composite key: unit_code + date_checked', () => {
|
||||
it('should use unit_code and date_checked as the filter for upsert', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'E101', price: 1200 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({
|
||||
unit_code: 'E101',
|
||||
date_checked: TODAY
|
||||
});
|
||||
|
||||
expect(doc).not.toBeNull();
|
||||
expect(doc.unit_code).toBe('E101');
|
||||
expect(doc.date_checked).toBe(TODAY);
|
||||
});
|
||||
|
||||
it('should create separate records for the same unit on different days', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'E201', price: 1200 }];
|
||||
|
||||
await insertPrices(db, units, '2026-02-04', logger);
|
||||
await insertPrices(db, units, '2026-02-05', logger);
|
||||
|
||||
const docs = await db.collection(PRICES_COLLECTION)
|
||||
.find({ unit_code: 'E201' })
|
||||
.sort({ date_checked: 1 })
|
||||
.toArray();
|
||||
|
||||
expect(docs).toHaveLength(2);
|
||||
expect(docs[0].date_checked).toBe('2026-02-04');
|
||||
expect(docs[1].date_checked).toBe('2026-02-05');
|
||||
});
|
||||
|
||||
it('should create separate records for different units on the same day', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'E301', price: 1000 },
|
||||
{ unit_code: 'E302', price: 1100 }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const docs = await db.collection(PRICES_COLLECTION)
|
||||
.find({ date_checked: TODAY })
|
||||
.toArray();
|
||||
|
||||
expect(docs).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Price record includes required fields
|
||||
// ---------------------------------------------------------------
|
||||
describe('price record fields', () => {
|
||||
it('should include unit_code in the price record', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'F101', price: 1400 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc.unit_code).toBe('F101');
|
||||
});
|
||||
|
||||
it('should include date_checked in the price record', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'F201', price: 1400 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc.date_checked).toBe(TODAY);
|
||||
});
|
||||
|
||||
it('should include price as a number in the price record', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'F301', price: 1550 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc.price).toBe(1550);
|
||||
expect(typeof doc.price).toBe('number');
|
||||
});
|
||||
|
||||
it('should include last_updated as an ISO timestamp string', async () => {
|
||||
const logger = createMockLogger();
|
||||
const beforeTime = getNow();
|
||||
|
||||
const units = [{ unit_code: 'F401', price: 1600 }];
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const afterTime = getNow();
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc.last_updated).toBeDefined();
|
||||
expect(typeof doc.last_updated).toBe('string');
|
||||
expect(doc.last_updated >= beforeTime).toBe(true);
|
||||
expect(doc.last_updated <= afterTime).toBe(true);
|
||||
});
|
||||
|
||||
it('should include data_source set to "web_scraper"', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'F501', price: 1700 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc.data_source).toBe('web_scraper');
|
||||
});
|
||||
|
||||
it('should include all five required fields in every price record', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'F601', price: 1800 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({});
|
||||
expect(doc).toHaveProperty('unit_code');
|
||||
expect(doc).toHaveProperty('date_checked');
|
||||
expect(doc).toHaveProperty('price');
|
||||
expect(doc).toHaveProperty('last_updated');
|
||||
expect(doc).toHaveProperty('data_source');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Empty price records logs warning and returns early
|
||||
// ---------------------------------------------------------------
|
||||
describe('empty price records', () => {
|
||||
it('should log a warning when no units have prices', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'G101', price: null },
|
||||
{ unit_code: 'G102', price: null }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
|
||||
});
|
||||
|
||||
it('should return insertedCount of 0 for all null-priced units', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'G201', price: null }];
|
||||
|
||||
const result = await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
expect(result.insertedCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should log a warning when units array is empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await insertPrices(db, [], TODAY, logger);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
|
||||
});
|
||||
|
||||
it('should not perform any database writes for empty/null-priced arrays', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await insertPrices(db, [], TODAY, logger);
|
||||
|
||||
const count = await db.collection(PRICES_COLLECTION).countDocuments();
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. Error handling when bulkWrite fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw error when bulkWrite fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'H101', price: 1000 }];
|
||||
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
|
||||
it('should log error details when bulkWrite fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'H201', price: 1000 }];
|
||||
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
try {
|
||||
await insertPrices(mockDb, units, TODAY, logger);
|
||||
} catch (e) {
|
||||
// Expected to throw
|
||||
}
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to insert prices',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should re-throw the original error', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'H301', price: 1000 }];
|
||||
|
||||
const originalError = new TypeError('Invalid operation');
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(originalError)
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Logging of inserted and updated counts
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging of result counts', () => {
|
||||
it('should log inserted and updated counts on success', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'I101', price: 1000 },
|
||||
{ unit_code: 'I102', price: 1100 }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Prices inserted',
|
||||
expect.objectContaining({
|
||||
inserted: expect.any(Number),
|
||||
updated: expect.any(Number)
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log correct counts for new inserts', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'I201', price: 1000 },
|
||||
{ unit_code: 'I202', price: 1100 }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// For new inserts: inserted=2, updated=0
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Prices inserted',
|
||||
expect.objectContaining({
|
||||
inserted: 2,
|
||||
updated: 0
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log correct counts when updating existing records', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'I301', price: 1000 }];
|
||||
|
||||
// First insert
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// Reset mock
|
||||
logger.info.mockClear();
|
||||
|
||||
// Second run updates existing
|
||||
const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
|
||||
await insertPrices(db, updatedUnits, TODAY, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Prices inserted',
|
||||
expect.objectContaining({
|
||||
inserted: 0,
|
||||
updated: 1
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. Re-running on same day updates existing records (idempotent)
|
||||
// ---------------------------------------------------------------
|
||||
describe('idempotent re-runs on same day', () => {
|
||||
it('should update the price when re-running on the same day', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First run with original price
|
||||
await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
|
||||
|
||||
const doc1 = await db.collection(PRICES_COLLECTION).findOne({
|
||||
unit_code: 'J101',
|
||||
date_checked: TODAY
|
||||
});
|
||||
expect(doc1.price).toBe(1200);
|
||||
|
||||
// Re-run with updated price (same day)
|
||||
await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
|
||||
|
||||
const doc2 = await db.collection(PRICES_COLLECTION).findOne({
|
||||
unit_code: 'J101',
|
||||
date_checked: TODAY
|
||||
});
|
||||
expect(doc2.price).toBe(1250);
|
||||
|
||||
// Should still have only 1 record
|
||||
const count = await db.collection(PRICES_COLLECTION).countDocuments({
|
||||
unit_code: 'J101',
|
||||
date_checked: TODAY
|
||||
});
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
|
||||
it('should update last_updated when re-running on the same day', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First run
|
||||
await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
|
||||
const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
|
||||
const firstUpdated = doc1.last_updated;
|
||||
|
||||
// Small delay to ensure different timestamp
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
// Re-run on same day
|
||||
await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
|
||||
const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
|
||||
const secondUpdated = doc2.last_updated;
|
||||
|
||||
expect(secondUpdated > firstUpdated).toBe(true);
|
||||
});
|
||||
|
||||
it('should return combined insert/update count reflecting the operation', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'J301', price: 1000 },
|
||||
{ unit_code: 'J302', price: 1100 }
|
||||
];
|
||||
|
||||
// First run - inserts
|
||||
const result1 = await insertPrices(db, units, TODAY, logger);
|
||||
expect(result1.insertedCount).toBe(2);
|
||||
|
||||
// Re-run - updates
|
||||
const result2 = await insertPrices(db, units, TODAY, logger);
|
||||
expect(result2.insertedCount).toBe(2); // combined upserted + modified
|
||||
|
||||
// Still only 2 total records
|
||||
const count = await db.collection(PRICES_COLLECTION).countDocuments();
|
||||
expect(count).toBe(2);
|
||||
});
|
||||
|
||||
it('should handle a mix of new and existing records on re-run', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First run with unit A
|
||||
await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
|
||||
|
||||
// Re-run with unit A (existing) and unit B (new)
|
||||
const mixedUnits = [
|
||||
{ unit_code: 'J401', price: 1050 },
|
||||
{ unit_code: 'J402', price: 1200 }
|
||||
];
|
||||
await insertPrices(db, mixedUnits, TODAY, logger);
|
||||
|
||||
// Should have 2 total records
|
||||
const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(2);
|
||||
|
||||
// Verify updated price
|
||||
const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
|
||||
expect(unitA.price).toBe(1050);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 9. Uses ordered: false for performance
|
||||
// ---------------------------------------------------------------
|
||||
describe('ordered: false for performance', () => {
|
||||
it('should successfully process all priced units in a batch', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'K101', price: 900 },
|
||||
{ unit_code: 'K102', price: 1000 },
|
||||
{ unit_code: 'K103', price: 1100 },
|
||||
{ unit_code: 'K104', price: 1200 },
|
||||
{ unit_code: 'K105', price: 1300 }
|
||||
];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
const count = await db.collection(PRICES_COLLECTION).countDocuments();
|
||||
expect(count).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 10. Uses correct collection
|
||||
// ---------------------------------------------------------------
|
||||
describe('collection usage', () => {
|
||||
it('should write to the configured PRICES collection', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'L101', price: 1400 }];
|
||||
|
||||
await insertPrices(db, units, TODAY, logger);
|
||||
|
||||
// Verify data is in the correct collection
|
||||
const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
|
||||
expect(doc).not.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,449 +0,0 @@
|
||||
/**
|
||||
* Tests for markStaleUnits() availability tracking
|
||||
*
|
||||
* Covers:
|
||||
* - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
|
||||
* - $set updates available to false and sets marked_stale_date
|
||||
* - Units in currentCodes are NOT marked stale
|
||||
* - Already unavailable units are not modified
|
||||
* - Error handling when updateMany fails
|
||||
* - Logging of modified count
|
||||
* - Empty currentUnitCodes handles gracefully
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require markStaleUnits after implementation
|
||||
let markStaleUnits;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
const UNITS_COLLECTION = 'units_scraper';
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ markStaleUnits } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('markStaleUnits', () => {
|
||||
// ---------------------------------------------------------------
|
||||
// 1. updateMany is called with correct filter
|
||||
// (unit_code $nin currentCodes AND available: true)
|
||||
// ---------------------------------------------------------------
|
||||
describe('filter criteria', () => {
|
||||
it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Seed units in the database
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'A101', available: true, price: 1200 },
|
||||
{ unit_code: 'A102', available: true, price: 1300 },
|
||||
{ unit_code: 'B201', available: true, price: 1500 }
|
||||
]);
|
||||
|
||||
// Current scrape found A101 and A102, but NOT B201
|
||||
const currentCodes = new Set(['A101', 'A102']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// B201 should be marked stale (it was available but not in current scrape)
|
||||
expect(result.modifiedCount).toBe(1);
|
||||
|
||||
// Verify B201 was updated
|
||||
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
|
||||
expect(staleUnit.available).toBe(false);
|
||||
expect(staleUnit.marked_stale_date).toBe(date);
|
||||
});
|
||||
|
||||
it('should only affect units NOT in the currentUnitCodes set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'X1', available: true },
|
||||
{ unit_code: 'X2', available: true },
|
||||
{ unit_code: 'X3', available: true },
|
||||
{ unit_code: 'X4', available: true }
|
||||
]);
|
||||
|
||||
// Current scrape found X1 and X3
|
||||
const currentCodes = new Set(['X1', 'X3']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// X1 and X3 should still be available
|
||||
const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
|
||||
const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
|
||||
expect(x1.available).toBe(true);
|
||||
expect(x3.available).toBe(true);
|
||||
|
||||
// X2 and X4 should be marked stale
|
||||
const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
|
||||
const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
|
||||
expect(x2.available).toBe(false);
|
||||
expect(x4.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. $set updates available to false and sets marked_stale_date
|
||||
// ---------------------------------------------------------------
|
||||
describe('$set fields', () => {
|
||||
it('should set available to false for stale units', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'S100', available: true, price: 1000 },
|
||||
{ unit_code: 'S200', available: true, price: 1100 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['S100']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
|
||||
expect(staleUnit.available).toBe(false);
|
||||
});
|
||||
|
||||
it('should set marked_stale_date to the provided date', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'D100', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
|
||||
expect(doc.marked_stale_date).toBe('2026-02-05');
|
||||
});
|
||||
|
||||
it('should update both available and marked_stale_date in the same operation', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'C100', available: true, price: 900 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-01-15';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
|
||||
expect(doc.available).toBe(false);
|
||||
expect(doc.marked_stale_date).toBe('2026-01-15');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Units in currentCodes are NOT marked stale
|
||||
// ---------------------------------------------------------------
|
||||
describe('units in currentCodes not affected', () => {
|
||||
it('should not modify units that are in the currentUnitCodes set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'KEEP-1', available: true, price: 1200 },
|
||||
{ unit_code: 'KEEP-2', available: true, price: 1300 },
|
||||
{ unit_code: 'STALE-1', available: true, price: 1400 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// Kept units should retain their original state
|
||||
const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
|
||||
const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
|
||||
|
||||
expect(keep1.available).toBe(true);
|
||||
expect(keep1.marked_stale_date).toBeUndefined();
|
||||
expect(keep2.available).toBe(true);
|
||||
expect(keep2.marked_stale_date).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'ALL-1', available: true },
|
||||
{ unit_code: 'ALL-2', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['ALL-1', 'ALL-2']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Already unavailable units are not modified
|
||||
// ---------------------------------------------------------------
|
||||
describe('already unavailable units', () => {
|
||||
it('should not modify units that are already unavailable (available: false)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
|
||||
{ unit_code: 'UNAVAIL-2', available: false },
|
||||
{ unit_code: 'AVAIL-1', available: true }
|
||||
]);
|
||||
|
||||
// None of these units are in the current scrape
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// Only AVAIL-1 should be modified (the already-unavailable ones should not be)
|
||||
expect(result.modifiedCount).toBe(1);
|
||||
|
||||
// Verify the already-unavailable unit's date was NOT changed
|
||||
const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
|
||||
expect(unavail1.marked_stale_date).toBe('2026-01-01');
|
||||
});
|
||||
|
||||
it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'U1', available: false },
|
||||
{ unit_code: 'U2', available: false }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Error handling when updateMany fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw error when updateMany fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
|
||||
it('should log error details when updateMany fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
try {
|
||||
await markStaleUnits(mockDb, currentCodes, date, logger);
|
||||
} catch (e) {
|
||||
// Expected to throw
|
||||
}
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to mark stale units',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should re-throw the original error', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const originalError = new TypeError('Invalid operation');
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(originalError)
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. Logging of modified count
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log the count of stale units marked', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'LOG-1', available: true },
|
||||
{ unit_code: 'LOG-2', available: true },
|
||||
{ unit_code: 'LOG-3', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['LOG-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 2 })
|
||||
);
|
||||
});
|
||||
|
||||
it('should log count of 0 when no units are marked stale', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'ZERO-1', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['ZERO-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 0 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Empty currentUnitCodes handles gracefully
|
||||
// ---------------------------------------------------------------
|
||||
describe('empty currentUnitCodes', () => {
|
||||
it('should mark all available units as stale when currentUnitCodes is empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'EMPTY-1', available: true },
|
||||
{ unit_code: 'EMPTY-2', available: true },
|
||||
{ unit_code: 'EMPTY-3', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(result.modifiedCount).toBe(3);
|
||||
|
||||
// All should be marked unavailable
|
||||
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
|
||||
for (const doc of docs) {
|
||||
expect(doc.available).toBe(false);
|
||||
expect(doc.marked_stale_date).toBe('2026-02-05');
|
||||
}
|
||||
});
|
||||
|
||||
it('should not throw when currentUnitCodes is an empty Set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
// Should not throw even with empty collection and empty set
|
||||
await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('should handle empty database with empty currentUnitCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 0 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. Return value
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value', () => {
|
||||
it('should return the MongoDB updateMany result', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'RET-1', available: true },
|
||||
{ unit_code: 'RET-2', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['RET-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// updateMany result should have these standard properties
|
||||
expect(result).toHaveProperty('matchedCount');
|
||||
expect(result).toHaveProperty('modifiedCount');
|
||||
expect(result).toHaveProperty('acknowledged');
|
||||
expect(result.acknowledged).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,517 +0,0 @@
|
||||
/**
|
||||
* Tests for parseUnits() - HTML parsing with cheerio
|
||||
*
|
||||
* Covers:
|
||||
* - Extraction of all data-spaces-* attributes
|
||||
* - Image URL extraction from nested img elements
|
||||
* - Container detection (section.spaces__tab-unit)
|
||||
* - Empty / missing container handling
|
||||
* - Deduplication by unit_code
|
||||
* - Logging behavior (info, warn, error)
|
||||
* - Missing attribute handling
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { parseUnits } = require('../../services/scraperService');
|
||||
|
||||
const fixturesDir = path.join(__dirname, 'fixtures');
|
||||
|
||||
// Helper: create a mock logger
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
describe('parseUnits', () => {
|
||||
let sampleHtml;
|
||||
let mockLogger;
|
||||
|
||||
beforeAll(() => {
|
||||
sampleHtml = fs.readFileSync(
|
||||
path.join(fixturesDir, 'sample-listing.html'),
|
||||
'utf-8'
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger = createMockLogger();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. Container detection
|
||||
// ---------------------------------------------------------------
|
||||
describe('container detection', () => {
|
||||
it('should find section.spaces__tab-unit container in valid HTML', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
// Should not log an error about missing container
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
expect(units.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should return empty array when section container is missing', () => {
|
||||
const html = '<html><body><div>No units here</div></body></html>';
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Container section.spaces__tab-unit not found'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
|
||||
it('should return empty array for empty HTML string', () => {
|
||||
const units = parseUnits('', mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return empty array when container exists but has no articles', () => {
|
||||
const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
// Container found, no error about container
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. Extraction of all data-spaces-* attributes
|
||||
// ---------------------------------------------------------------
|
||||
describe('attribute extraction', () => {
|
||||
it('should extract all expected attributes from a fully populated article', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit1 = units.find(u => u.unit_code === 'W2707');
|
||||
|
||||
expect(unit1).toBeDefined();
|
||||
|
||||
// Core identifiers
|
||||
expect(unit1.id).toBe('154842');
|
||||
expect(unit1.unit_code).toBe('W2707');
|
||||
expect(unit1.unit_id).toBe('154842');
|
||||
|
||||
// Physical attributes
|
||||
expect(unit1.floor).toBe('2755');
|
||||
expect(unit1.area).toBe('1210');
|
||||
expect(unit1.bed_count).toBe('2');
|
||||
expect(unit1.bath_count).toBe('2');
|
||||
|
||||
// Pricing
|
||||
expect(unit1.price).toBe('5230');
|
||||
|
||||
// Availability
|
||||
expect(unit1.available).toBe('true');
|
||||
expect(unit1.unavailable).toBe('false');
|
||||
expect(unit1.soonest).toBe('2025-10-11');
|
||||
expect(unit1.date_available).toBe('1760140800');
|
||||
|
||||
// Plan information
|
||||
expect(unit1.plan_id).toBe('10270');
|
||||
expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
|
||||
|
||||
// Property information
|
||||
expect(unit1.obj_type).toBe('unit');
|
||||
expect(unit1.community).toBe('Country Club Towers');
|
||||
expect(unit1.asset).toBe('420');
|
||||
|
||||
// URLs
|
||||
expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
|
||||
expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
|
||||
|
||||
// Specials
|
||||
expect(unit1.specials_content).toBe('');
|
||||
});
|
||||
|
||||
it('should extract correct count of units from the fixture', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
expect(units).toHaveLength(10);
|
||||
});
|
||||
|
||||
it('should return raw string values without type conversion', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit = units.find(u => u.unit_code === 'W2707');
|
||||
|
||||
// All values should be strings (raw extraction, no conversion)
|
||||
expect(typeof unit.id).toBe('string');
|
||||
expect(typeof unit.price).toBe('string');
|
||||
expect(typeof unit.floor).toBe('string');
|
||||
expect(typeof unit.available).toBe('string');
|
||||
expect(typeof unit.bed_count).toBe('string');
|
||||
expect(typeof unit.bath_count).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Image URL extraction
|
||||
// ---------------------------------------------------------------
|
||||
describe('image URL extraction', () => {
|
||||
it('should extract image_url from img src attribute', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
|
||||
<img src="https://example.com/images/unit-1.jpg" />
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
|
||||
});
|
||||
|
||||
it('should fall back to data-src when src is not present', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
|
||||
<img data-src="https://example.com/images/unit-2.jpg" />
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
|
||||
});
|
||||
|
||||
it('should not set image_url when no img element exists', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
|
||||
<div>No image here</div>
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units[0].image_url).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Missing attributes
|
||||
// ---------------------------------------------------------------
|
||||
describe('missing attributes', () => {
|
||||
it('should return undefined for attributes not present on the article', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article
|
||||
data-spaces-id="1005"
|
||||
data-spaces-unit="SPARSE-1"
|
||||
data-spaces-unit-id="5005"
|
||||
data-spaces-sort-price="1300"
|
||||
data-spaces-available="true"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100">
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
const unit = units[0];
|
||||
|
||||
expect(unit).toBeDefined();
|
||||
expect(unit.floor).toBeUndefined();
|
||||
expect(unit.area).toBeUndefined();
|
||||
expect(unit.bed_count).toBeUndefined();
|
||||
expect(unit.bath_count).toBeUndefined();
|
||||
expect(unit.soonest).toBeUndefined();
|
||||
expect(unit.date_available).toBeUndefined();
|
||||
expect(unit.plan_id).toBeUndefined();
|
||||
expect(unit.plan_name).toBeUndefined();
|
||||
expect(unit.href).toBeUndefined();
|
||||
expect(unit.inventory_href).toBeUndefined();
|
||||
expect(unit.specials_content).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should still extract the attributes that are present on a sparse article', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article
|
||||
data-spaces-id="1005"
|
||||
data-spaces-unit="SPARSE-1"
|
||||
data-spaces-unit-id="5005"
|
||||
data-spaces-sort-price="1300"
|
||||
data-spaces-available="true"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100">
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
const unit = units[0];
|
||||
|
||||
expect(unit.id).toBe('1005');
|
||||
expect(unit.unit_code).toBe('SPARSE-1');
|
||||
expect(unit.unit_id).toBe('5005');
|
||||
expect(unit.price).toBe('1300');
|
||||
expect(unit.available).toBe('true');
|
||||
expect(unit.obj_type).toBe('unit');
|
||||
expect(unit.community).toBe('Country Club Towers');
|
||||
expect(unit.asset).toBe('100');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Deduplication
|
||||
// ---------------------------------------------------------------
|
||||
describe('deduplication', () => {
|
||||
it('should deduplicate units with the same unit_code', () => {
|
||||
// Build HTML with duplicate articles
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
const codes = units.map(u => u.unit_code);
|
||||
|
||||
// Should have 2 unique unit codes
|
||||
expect(units).toHaveLength(2);
|
||||
expect(new Set(codes).size).toBe(codes.length);
|
||||
});
|
||||
|
||||
it('should keep the first occurrence when deduplicating', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
// First occurrence (id=1, price=1000) should be kept
|
||||
expect(units[0].id).toBe('1');
|
||||
expect(units[0].price).toBe('1000');
|
||||
});
|
||||
|
||||
it('should log a warning when duplicates are removed', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
parseUnits(html, mockLogger);
|
||||
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(
|
||||
'Duplicate units removed',
|
||||
expect.objectContaining({
|
||||
original: 3,
|
||||
deduplicated: 2
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should not log a warning when there are no duplicates', () => {
|
||||
parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
// The fixture has no duplicates, so no warn about duplicates
|
||||
const warnCalls = mockLogger.warn.mock.calls;
|
||||
const duplicateWarns = warnCalls.filter(
|
||||
call => call[0] === 'Duplicate units removed'
|
||||
);
|
||||
expect(duplicateWarns).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should filter out articles with no unit_code during deduplication', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
// Article without unit_code should be filtered out
|
||||
expect(units).toHaveLength(1);
|
||||
expect(units[0].unit_code).toBe('VALID-1');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. Logging behavior
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log the count of parsed units', () => {
|
||||
parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Units parsed',
|
||||
expect.objectContaining({ count: 10 })
|
||||
);
|
||||
});
|
||||
|
||||
it('should log error when container is not found', () => {
|
||||
const html = '<html><body><p>Nothing here</p></body></html>';
|
||||
parseUnits(html, mockLogger);
|
||||
|
||||
expect(mockLogger.error).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Container section.spaces__tab-unit not found'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
|
||||
// ---------------------------------------------------------------
|
||||
describe('fixture: sample-listing-empty.html', () => {
|
||||
let emptyHtml;
|
||||
|
||||
beforeAll(() => {
|
||||
emptyHtml = fs.readFileSync(
|
||||
path.join(fixturesDir, 'sample-listing-empty.html'),
|
||||
'utf-8'
|
||||
);
|
||||
});
|
||||
|
||||
it('should return empty array when fixture has container but no articles', () => {
|
||||
const units = parseUnits(emptyHtml, mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
// Container exists so no error about missing container
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should log zero parsed units for empty fixture', () => {
|
||||
parseUnits(emptyHtml, mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Units parsed',
|
||||
expect.objectContaining({ count: 0 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fixture: sample-listing-call.html', () => {
|
||||
let callHtml;
|
||||
|
||||
beforeAll(() => {
|
||||
callHtml = fs.readFileSync(
|
||||
path.join(fixturesDir, 'sample-listing-call.html'),
|
||||
'utf-8'
|
||||
);
|
||||
});
|
||||
|
||||
it('should extract both units from the call-for-pricing fixture', () => {
|
||||
const units = parseUnits(callHtml, mockLogger);
|
||||
|
||||
expect(units).toHaveLength(2);
|
||||
expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
|
||||
});
|
||||
|
||||
it('should preserve "Call for pricing" as raw string in the price field', () => {
|
||||
const units = parseUnits(callHtml, mockLogger);
|
||||
|
||||
expect(units[0].price).toBe('Call for pricing');
|
||||
expect(units[1].price).toBe('Call for pricing');
|
||||
});
|
||||
|
||||
it('should extract all attributes correctly from call-for-pricing units', () => {
|
||||
const units = parseUnits(callHtml, mockLogger);
|
||||
const unit = units.find(u => u.unit_code === 'P-101');
|
||||
|
||||
expect(unit.id).toBe('9001');
|
||||
expect(unit.unit_id).toBe('9001');
|
||||
expect(unit.floor).toBe('1');
|
||||
expect(unit.area).toBe('750');
|
||||
expect(unit.bed_count).toBe('1');
|
||||
expect(unit.bath_count).toBe('1');
|
||||
expect(unit.available).toBe('true');
|
||||
expect(unit.unavailable).toBe('false');
|
||||
expect(unit.soonest).toBe('Now');
|
||||
expect(unit.date_available).toBe('1706745600');
|
||||
expect(unit.plan_id).toBe('500');
|
||||
expect(unit.plan_name).toBe('Penthouse A');
|
||||
expect(unit.community).toBe('Country Club Towers');
|
||||
expect(unit.asset).toBe('420');
|
||||
expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
|
||||
expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
|
||||
expect(unit.specials_content).toBe('');
|
||||
});
|
||||
|
||||
it('should not have image_url when fixture units lack img elements', () => {
|
||||
const units = parseUnits(callHtml, mockLogger);
|
||||
|
||||
expect(units[0].image_url).toBeUndefined();
|
||||
expect(units[1].image_url).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. Multiple units with varied data
|
||||
// ---------------------------------------------------------------
|
||||
describe('varied unit data', () => {
|
||||
it('should handle unavailable units correctly', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article
|
||||
data-spaces-id="4001"
|
||||
data-spaces-unit="UNAVAIL-1"
|
||||
data-spaces-sort-price="2000"
|
||||
data-spaces-available="false"
|
||||
data-spaces-unavailable="true"
|
||||
data-spaces-soonest=""
|
||||
data-spaces-sort-date="">
|
||||
</article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
const units = parseUnits(html, mockLogger);
|
||||
const unit = units[0];
|
||||
|
||||
expect(unit.available).toBe('false');
|
||||
expect(unit.unavailable).toBe('true');
|
||||
expect(unit.soonest).toBe('');
|
||||
expect(unit.date_available).toBe('');
|
||||
});
|
||||
|
||||
it('should handle half bath counts', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
// E3205 is the penthouse with 2.5 baths
|
||||
const unit = units.find(u => u.unit_code === 'E3205');
|
||||
|
||||
expect(unit.bath_count).toBe('2.5');
|
||||
});
|
||||
|
||||
it('should handle studio units (bed_count = 0)', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
// W2603 is the studio with bed_count 0
|
||||
const unit = units.find(u => u.unit_code === 'W2603');
|
||||
|
||||
expect(unit.bed_count).toBe('0');
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,308 +0,0 @@
|
||||
/**
|
||||
* Tests for recordScraperRun() history persistence
|
||||
*
|
||||
* Covers:
|
||||
* - insertOne is called with runData plus recordedAt timestamp
|
||||
* - All runData fields are preserved (jobId, trigger, status, duration, etc.)
|
||||
* - Error is caught and logged but NOT thrown (graceful failure)
|
||||
* - Returns null on error instead of crashing
|
||||
* - Returns insert result on success
|
||||
* - recordedAt is a valid Date object
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require after implementation
|
||||
let recordScraperRun;
|
||||
let getNow;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
let logger;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ recordScraperRun, getNow } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean all collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
|
||||
// Create fresh mock logger for each test
|
||||
logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
});
|
||||
|
||||
describe('recordScraperRun', () => {
|
||||
const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
|
||||
|
||||
// Sample runData matching the structure from runScrape()'s finally block
|
||||
function createSampleRunData(overrides = {}) {
|
||||
return {
|
||||
jobId: 'test-job-001',
|
||||
trigger: 'manual',
|
||||
dryRun: false,
|
||||
status: 'success',
|
||||
startedAt: '2026-02-06T06:00:00.000Z',
|
||||
completedAt: '2026-02-06T06:00:12.345Z',
|
||||
duration: 12345,
|
||||
unitsProcessed: 50,
|
||||
pricesInserted: 48,
|
||||
newUnitsCount: 3,
|
||||
rentedUnitsCount: 1,
|
||||
staleUnitsCount: 2,
|
||||
errors: [],
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. insertOne is called with runData plus recordedAt timestamp
|
||||
// ---------------------------------------------------------------
|
||||
describe('insertOne with runData and recordedAt', () => {
|
||||
it('should insert a document into the scraper_runs collection', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should include a recordedAt field in the inserted document', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.recordedAt).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. All runData fields are preserved
|
||||
// ---------------------------------------------------------------
|
||||
describe('preserving all runData fields', () => {
|
||||
it('should preserve jobId, trigger, status, and duration', async () => {
|
||||
const runData = createSampleRunData({
|
||||
jobId: 'preserve-test-001',
|
||||
trigger: 'scheduled',
|
||||
status: 'failed',
|
||||
duration: 9999
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.jobId).toBe('preserve-test-001');
|
||||
expect(doc.trigger).toBe('scheduled');
|
||||
expect(doc.status).toBe('failed');
|
||||
expect(doc.duration).toBe(9999);
|
||||
});
|
||||
|
||||
it('should preserve unit processing metrics', async () => {
|
||||
const runData = createSampleRunData({
|
||||
unitsProcessed: 42,
|
||||
pricesInserted: 40,
|
||||
newUnitsCount: 5,
|
||||
rentedUnitsCount: 2,
|
||||
staleUnitsCount: 3
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.unitsProcessed).toBe(42);
|
||||
expect(doc.pricesInserted).toBe(40);
|
||||
expect(doc.newUnitsCount).toBe(5);
|
||||
expect(doc.rentedUnitsCount).toBe(2);
|
||||
expect(doc.staleUnitsCount).toBe(3);
|
||||
});
|
||||
|
||||
it('should preserve timing fields (startedAt, completedAt)', async () => {
|
||||
const runData = createSampleRunData({
|
||||
startedAt: '2026-02-06T10:00:00.000Z',
|
||||
completedAt: '2026-02-06T10:00:15.500Z'
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
|
||||
expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
|
||||
});
|
||||
|
||||
it('should preserve dryRun flag', async () => {
|
||||
const runData = createSampleRunData({ dryRun: true });
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.dryRun).toBe(true);
|
||||
});
|
||||
|
||||
it('should preserve errors array when it has entries', async () => {
|
||||
const runData = createSampleRunData({
|
||||
status: 'failed',
|
||||
errors: ['Connection timeout', 'Retry exhausted']
|
||||
});
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
|
||||
});
|
||||
|
||||
it('should preserve empty errors array for successful runs', async () => {
|
||||
const runData = createSampleRunData({ errors: [] });
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Error is caught and logged but NOT thrown (graceful failure)
|
||||
// ---------------------------------------------------------------
|
||||
describe('graceful error handling', () => {
|
||||
it('should NOT throw when insertOne fails', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
// Create a mock db that throws on insertOne
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
// This should NOT throw
|
||||
await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should log error message when insert fails', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await recordScraperRun(mockDb, runData, logger);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to record scraper run',
|
||||
{ errorMessage: 'Disk full' }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Returns null on error instead of crashing
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value on error', () => {
|
||||
it('should return null when insertOne throws', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const mockCollection = {
|
||||
insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const result = await recordScraperRun(mockDb, runData, logger);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Returns insert result on success
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value on success', () => {
|
||||
it('should return the insertOne result object', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('should return result with acknowledged property', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result.acknowledged).toBe(true);
|
||||
});
|
||||
|
||||
it('should return result with insertedId', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
const result = await recordScraperRun(db, runData, logger);
|
||||
|
||||
expect(result.insertedId).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. recordedAt is a valid Date object
|
||||
// ---------------------------------------------------------------
|
||||
describe('recordedAt field', () => {
|
||||
it('should set recordedAt as a string timestamp', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(typeof doc.recordedAt).toBe('string');
|
||||
});
|
||||
|
||||
it('should set recordedAt close to current time', async () => {
|
||||
const beforeTime = getNow();
|
||||
|
||||
const runData = createSampleRunData();
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const afterTime = getNow();
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
expect(doc.recordedAt >= beforeTime).toBe(true);
|
||||
expect(doc.recordedAt <= afterTime).toBe(true);
|
||||
});
|
||||
|
||||
it('should not overwrite any existing runData fields with recordedAt', async () => {
|
||||
const runData = createSampleRunData();
|
||||
|
||||
await recordScraperRun(db, runData, logger);
|
||||
|
||||
const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
|
||||
|
||||
// Verify the original fields still exist alongside recordedAt
|
||||
expect(doc.jobId).toBe(runData.jobId);
|
||||
expect(doc.status).toBe(runData.status);
|
||||
expect(typeof doc.recordedAt).toBe('string');
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,192 +0,0 @@
|
||||
/**
|
||||
* Tests for config/scraper.js
|
||||
* Tests scraper configuration constants and environment variable overrides
|
||||
*/
|
||||
|
||||
describe('config/scraper', () => {
|
||||
// Store original env values to restore after tests
|
||||
const originalEnv = { ...process.env };
|
||||
|
||||
beforeEach(() => {
|
||||
// Clear all scraper-related env vars before each test
|
||||
delete process.env.SCRAPER_SCHEDULE;
|
||||
delete process.env.SCRAPER_TIMEZONE;
|
||||
delete process.env.SCRAPER_ENABLED;
|
||||
delete process.env.SCRAPER_TIMEOUT;
|
||||
delete process.env.SCRAPER_UNITS_COLLECTION;
|
||||
delete process.env.SCRAPER_PRICES_COLLECTION;
|
||||
delete process.env.SCRAPER_SUMMARIES_COLLECTION;
|
||||
delete process.env.SCRAPER_RUNS_COLLECTION;
|
||||
|
||||
// Clear the require cache to reload config with fresh env
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
// Restore original environment
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
describe('TARGET_URL', () => {
|
||||
test('should be set to the correct apartment listing URL', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.TARGET_URL).toBe(
|
||||
'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_SCHEDULE', () => {
|
||||
test('should default to "0 6 * * *" when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_SCHEDULE = '0 12 * * *';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_TIMEZONE', () => {
|
||||
test('should default to "America/Denver" when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_TIMEZONE = 'Europe/London';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEZONE).toBe('Europe/London');
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_ENABLED', () => {
|
||||
test('should default to true when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
|
||||
test('should be false when env var is set to "false"', () => {
|
||||
process.env.SCRAPER_ENABLED = 'false';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(false);
|
||||
});
|
||||
|
||||
test('should be true when env var is set to any other value', () => {
|
||||
process.env.SCRAPER_ENABLED = 'true';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
|
||||
test('should be true when env var is empty string', () => {
|
||||
process.env.SCRAPER_ENABLED = '';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_TIMEOUT', () => {
|
||||
test('should default to 30000 when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(30000);
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '60000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(60000);
|
||||
});
|
||||
|
||||
test('should parse string to integer', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '45000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(45000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('USER_AGENT', () => {
|
||||
test('should be a valid User-Agent string', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('RETRY_CONFIG', () => {
|
||||
test('should have maxRetries property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
|
||||
expect(config.RETRY_CONFIG.maxRetries).toBe(3);
|
||||
});
|
||||
|
||||
test('should have baseDelay property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
|
||||
expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
|
||||
});
|
||||
|
||||
test('should have timeout property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('timeout');
|
||||
expect(config.RETRY_CONFIG.timeout).toBe(30000);
|
||||
});
|
||||
|
||||
test('timeout should respect SCRAPER_TIMEOUT env var', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '45000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG.timeout).toBe(45000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('COLLECTIONS', () => {
|
||||
describe('default values', () => {
|
||||
test('UNITS should default to "units_scraper"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
|
||||
});
|
||||
|
||||
test('PRICES should default to "unit_prices_scraper"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
|
||||
});
|
||||
|
||||
test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
|
||||
});
|
||||
|
||||
test('SCRAPER_RUNS should default to "scraper_runs"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('env var overrides', () => {
|
||||
test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.UNITS).toBe('units_custom');
|
||||
});
|
||||
|
||||
test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
|
||||
});
|
||||
|
||||
test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
|
||||
});
|
||||
|
||||
test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,471 +0,0 @@
|
||||
/**
|
||||
* Tests for scraperLogger.js
|
||||
* Following TDD - these tests are written BEFORE implementation
|
||||
*/
|
||||
|
||||
const { createLogger, LEVELS } = require('../../services/scraperLogger');
|
||||
|
||||
describe('scraperLogger', () => {
|
||||
let consoleSpy;
|
||||
let logOutput;
|
||||
|
||||
beforeEach(() => {
|
||||
logOutput = [];
|
||||
consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
|
||||
logOutput.push(output);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
|
||||
describe('createLogger', () => {
|
||||
test('should return an object with info, warn, error methods', () => {
|
||||
const logger = createLogger('test-job-id');
|
||||
|
||||
expect(typeof logger).toBe('object');
|
||||
expect(typeof logger.info).toBe('function');
|
||||
expect(typeof logger.warn).toBe('function');
|
||||
expect(typeof logger.error).toBe('function');
|
||||
});
|
||||
|
||||
test('should include jobId in all log entries', () => {
|
||||
const jobId = 'unique-job-123';
|
||||
const logger = createLogger(jobId);
|
||||
|
||||
logger.info('Test message');
|
||||
|
||||
expect(logOutput.length).toBe(1);
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.jobId).toBe(jobId);
|
||||
});
|
||||
});
|
||||
|
||||
describe('log entry format', () => {
|
||||
test('should include timestamp in ISO format', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const beforeTime = new Date().toISOString();
|
||||
|
||||
logger.info('Test message');
|
||||
|
||||
const afterTime = new Date().toISOString();
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
|
||||
expect(parsed.timestamp).toBeDefined();
|
||||
// Verify timestamp is in ISO format
|
||||
expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
|
||||
// Verify timestamp is within test window
|
||||
expect(parsed.timestamp >= beforeTime).toBe(true);
|
||||
expect(parsed.timestamp <= afterTime).toBe(true);
|
||||
});
|
||||
|
||||
test('should include log level', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Info message');
|
||||
logger.warn('Warn message');
|
||||
logger.error('Error message');
|
||||
|
||||
expect(logOutput.length).toBe(3);
|
||||
|
||||
const infoEntry = JSON.parse(logOutput[0]);
|
||||
const warnEntry = JSON.parse(logOutput[1]);
|
||||
const errorEntry = JSON.parse(logOutput[2]);
|
||||
|
||||
expect(infoEntry.level).toBe('info');
|
||||
expect(warnEntry.level).toBe('warn');
|
||||
expect(errorEntry.level).toBe('error');
|
||||
});
|
||||
|
||||
test('should include message', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Test message content');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.message).toBe('Test message content');
|
||||
});
|
||||
|
||||
test('should include context when provided', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = { key: 'value', count: 42 };
|
||||
|
||||
logger.info('Test message', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context).toEqual(context);
|
||||
});
|
||||
|
||||
test('should output valid JSON', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Test message', { data: 'test' });
|
||||
|
||||
expect(() => JSON.parse(logOutput[0])).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('log levels', () => {
|
||||
test('info() should output with level "info"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Info message');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.level).toBe('info');
|
||||
});
|
||||
|
||||
test('warn() should output with level "warn"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.warn('Warning message');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.level).toBe('warn');
|
||||
});
|
||||
|
||||
test('error() should output with level "error"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.error('Error message');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.level).toBe('error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('LEVELS constant', () => {
|
||||
test('should export LEVELS constant with info, warn, error', () => {
|
||||
expect(LEVELS).toBeDefined();
|
||||
expect(LEVELS.info).toBe('info');
|
||||
expect(LEVELS.warn).toBe('warn');
|
||||
expect(LEVELS.error).toBe('error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('message truncation', () => {
|
||||
test('should truncate messages longer than 1000 characters', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const longMessage = 'x'.repeat(1500);
|
||||
|
||||
logger.info(longMessage);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
|
||||
expect(parsed.message).toContain('... [truncated]');
|
||||
});
|
||||
|
||||
test('should not truncate messages of 1000 characters or less', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const shortMessage = 'x'.repeat(1000);
|
||||
|
||||
logger.info(shortMessage);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.message).toBe(shortMessage);
|
||||
expect(parsed.message).not.toContain('... [truncated]');
|
||||
});
|
||||
|
||||
test('should handle empty message', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.message).toBe('');
|
||||
});
|
||||
|
||||
test('should handle null or undefined message', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info(null);
|
||||
logger.info(undefined);
|
||||
|
||||
expect(logOutput.length).toBe(2);
|
||||
// Should not throw and should handle gracefully
|
||||
});
|
||||
});
|
||||
|
||||
describe('MongoDB connection string redaction', () => {
|
||||
test('should redact username and password from mongodb:// URIs', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
|
||||
};
|
||||
|
||||
logger.info('Database connection', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
|
||||
expect(parsed.context.uri).not.toContain('admin');
|
||||
expect(parsed.context.uri).not.toContain('secretpassword123');
|
||||
});
|
||||
|
||||
test('should redact username and password from mongodb+srv:// URIs', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
|
||||
};
|
||||
|
||||
logger.info('Database connection', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
|
||||
expect(parsed.context.uri).not.toContain('myuser');
|
||||
expect(parsed.context.uri).not.toContain('mypass');
|
||||
});
|
||||
|
||||
test('should handle connection string without credentials', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
uri: 'mongodb://localhost:27017/mydb'
|
||||
};
|
||||
|
||||
logger.info('Database connection', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitive key redaction', () => {
|
||||
test('should redact keys containing "password"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
password: 'secret123',
|
||||
userPassword: 'anotherSecret',
|
||||
password_hash: 'hashed'
|
||||
};
|
||||
|
||||
logger.info('User data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.password).toBe('[REDACTED]');
|
||||
expect(parsed.context.userPassword).toBe('[REDACTED]');
|
||||
expect(parsed.context.password_hash).toBe('[REDACTED]');
|
||||
});
|
||||
|
||||
test('should redact keys containing "secret"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
secret: 'mysecret',
|
||||
clientSecret: 'secret123',
|
||||
SECRET_KEY: 'key'
|
||||
};
|
||||
|
||||
logger.info('Config data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.secret).toBe('[REDACTED]');
|
||||
expect(parsed.context.clientSecret).toBe('[REDACTED]');
|
||||
// Case-insensitive check
|
||||
});
|
||||
|
||||
test('should redact keys containing "token"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
token: 'abc123',
|
||||
accessToken: 'token456',
|
||||
refresh_token: 'refresh789'
|
||||
};
|
||||
|
||||
logger.info('Auth data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.token).toBe('[REDACTED]');
|
||||
expect(parsed.context.accessToken).toBe('[REDACTED]');
|
||||
expect(parsed.context.refresh_token).toBe('[REDACTED]');
|
||||
});
|
||||
|
||||
test('should redact keys containing "apikey"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
apikey: 'key123',
|
||||
apiKey: 'key456',
|
||||
api_key: 'key789' // Note: underscores may not match 'apikey'
|
||||
};
|
||||
|
||||
logger.info('API data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.apikey).toBe('[REDACTED]');
|
||||
expect(parsed.context.apiKey).toBe('[REDACTED]');
|
||||
});
|
||||
|
||||
test('should redact keys containing "authorization"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
authorization: 'Bearer token123',
|
||||
Authorization: 'Basic base64string'
|
||||
};
|
||||
|
||||
logger.info('Header data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.authorization).toBe('[REDACTED]');
|
||||
});
|
||||
|
||||
test('should not redact non-sensitive keys', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
username: 'john',
|
||||
email: 'john@example.com',
|
||||
count: 42
|
||||
};
|
||||
|
||||
logger.info('User info', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.username).toBe('john');
|
||||
expect(parsed.context.email).toBe('john@example.com');
|
||||
expect(parsed.context.count).toBe(42);
|
||||
});
|
||||
});
|
||||
|
||||
describe('circular reference handling', () => {
|
||||
test('should handle circular references in context', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = { name: 'test' };
|
||||
context.self = context; // Create circular reference
|
||||
|
||||
// Should not throw
|
||||
expect(() => {
|
||||
logger.info('Circular test', context);
|
||||
}).not.toThrow();
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.name).toBe('test');
|
||||
expect(parsed.context.self).toBe('[Circular]');
|
||||
});
|
||||
|
||||
test('should handle deeply nested circular references', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
level1: {
|
||||
level2: {
|
||||
level3: {}
|
||||
}
|
||||
}
|
||||
};
|
||||
context.level1.level2.level3.back = context.level1;
|
||||
|
||||
expect(() => {
|
||||
logger.info('Deep circular test', context);
|
||||
}).not.toThrow();
|
||||
|
||||
// Should be valid JSON output
|
||||
expect(() => JSON.parse(logOutput[0])).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Buffer handling', () => {
|
||||
test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
data: Buffer.from('Hello World')
|
||||
};
|
||||
|
||||
logger.info('Buffer data', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
|
||||
});
|
||||
|
||||
test('should handle empty Buffer', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
data: Buffer.alloc(0)
|
||||
};
|
||||
|
||||
logger.info('Empty buffer', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
|
||||
});
|
||||
|
||||
test('should handle large Buffer', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
data: Buffer.alloc(1024 * 1024) // 1MB buffer
|
||||
};
|
||||
|
||||
logger.info('Large buffer', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
test('should handle empty context object', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Test message', {});
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context).toEqual({});
|
||||
});
|
||||
|
||||
test('should handle context not provided', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Test message');
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context).toEqual({});
|
||||
});
|
||||
|
||||
test('should handle null context', () => {
|
||||
const logger = createLogger('test-job');
|
||||
|
||||
logger.info('Test message', null);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context).toEqual({});
|
||||
});
|
||||
|
||||
test('should handle undefined values in context', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
defined: 'value',
|
||||
undefinedValue: undefined
|
||||
};
|
||||
|
||||
logger.info('Test message', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.defined).toBe('value');
|
||||
// undefined is typically converted to null in JSON
|
||||
expect(parsed.context.undefinedValue).toBeNull();
|
||||
});
|
||||
|
||||
test('should handle nested objects in context', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
outer: {
|
||||
inner: {
|
||||
value: 'deep'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
logger.info('Nested test', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.outer.inner.value).toBe('deep');
|
||||
});
|
||||
|
||||
test('should handle arrays in context', () => {
|
||||
const logger = createLogger('test-job');
|
||||
const context = {
|
||||
items: [1, 2, 3, 'four']
|
||||
};
|
||||
|
||||
logger.info('Array test', context);
|
||||
|
||||
const parsed = JSON.parse(logOutput[0]);
|
||||
expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,739 +0,0 @@
|
||||
/**
|
||||
* Tests for runScrape() orchestration function
|
||||
*
|
||||
* Covers:
|
||||
* - Full workflow completes with mocked dependencies
|
||||
* - Returns correct result structure with jobId, status, metrics
|
||||
* - Handles dryRun option (skip DB writes)
|
||||
* - Handles htmlContent option (use provided HTML)
|
||||
* - Records scraper run to history on success
|
||||
* - Records scraper run to history on failure
|
||||
* - Catches and logs errors from fetchPage
|
||||
* - Catches and logs errors from database operations
|
||||
* - Calculates newUnitsCount and rentedUnitsCount correctly
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
|
||||
let runScrape;
|
||||
let recordScraperRun;
|
||||
let createScraperIndexes;
|
||||
|
||||
// Store original module references so we can mock individual functions
|
||||
let scraperService;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
scraperService = require('../../services/scraperService');
|
||||
({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean all relevant collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
// Reset all mocks
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Helper: Create sample HTML with units
|
||||
// ============================================================
|
||||
function createSampleHtml(unitCodes) {
|
||||
const articles = unitCodes.map(code => `
|
||||
<article
|
||||
data-spaces-id="100"
|
||||
data-spaces-unit="${code}"
|
||||
data-spaces-unit-id="200"
|
||||
data-spaces-unit-floor="5"
|
||||
data-spaces-sort-area="750"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-sort-price="1500"
|
||||
data-spaces-available="true"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-soonest="Now"
|
||||
data-spaces-sort-date="1700000000"
|
||||
data-spaces-plan-id="10"
|
||||
data-spaces-sort-plan-name="Studio"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="TestCommunity"
|
||||
data-spaces-asset="1"
|
||||
data-spaces-href="/unit/${code}"
|
||||
data-spaces-inventory-href="/inventory/${code}"
|
||||
>
|
||||
<img src="http://example.com/${code}.jpg" />
|
||||
</article>
|
||||
`).join('\n');
|
||||
|
||||
return `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
${articles}
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Test: recordScraperRun
|
||||
// ============================================================
|
||||
describe('recordScraperRun', () => {
|
||||
it('should insert a run record into scraper_runs collection', async () => {
|
||||
const runData = {
|
||||
jobId: 'test-job-001',
|
||||
trigger: 'manual',
|
||||
status: 'success',
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
duration: 1234,
|
||||
unitsProcessed: 10,
|
||||
pricesInserted: 8,
|
||||
errors: []
|
||||
};
|
||||
|
||||
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
const result = await recordScraperRun(db, runData, mockLogger);
|
||||
|
||||
expect(result).toBeTruthy();
|
||||
expect(result.insertedId).toBeTruthy();
|
||||
|
||||
// Verify it was inserted
|
||||
const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
|
||||
expect(saved).toBeTruthy();
|
||||
expect(saved.status).toBe('success');
|
||||
expect(typeof saved.recordedAt).toBe('string');
|
||||
});
|
||||
|
||||
it('should not throw when insert fails', async () => {
|
||||
// Pass null db to cause an error
|
||||
const runData = { jobId: 'test-fail', status: 'success' };
|
||||
|
||||
// Should not throw
|
||||
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
const result = await recordScraperRun(null, runData, mockLogger);
|
||||
expect(result).toBeNull();
|
||||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: runScrape - Full workflow
|
||||
// ============================================================
|
||||
describe('runScrape', () => {
|
||||
it('should complete full workflow with mocked dependencies', async () => {
|
||||
const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
trigger: 'manual',
|
||||
jobId: 'test-full-workflow',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result).toBeTruthy();
|
||||
expect(result.jobId).toBe('test-full-workflow');
|
||||
expect(result.status).toBe('success');
|
||||
expect(result.trigger).toBe('manual');
|
||||
expect(result.unitsProcessed).toBe(2);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return correct result structure with jobId, status, metrics', async () => {
|
||||
const html = createSampleHtml(['UNIT-X']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-structure',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
// Verify all expected fields exist
|
||||
expect(result).toHaveProperty('jobId', 'test-structure');
|
||||
expect(result).toHaveProperty('trigger', 'manual');
|
||||
expect(result).toHaveProperty('dryRun', false);
|
||||
expect(result).toHaveProperty('status', 'success');
|
||||
expect(result).toHaveProperty('startedAt');
|
||||
expect(result).toHaveProperty('completedAt');
|
||||
expect(result).toHaveProperty('duration');
|
||||
expect(result).toHaveProperty('unitsProcessed');
|
||||
expect(result).toHaveProperty('pricesInserted');
|
||||
expect(result).toHaveProperty('newUnitsCount');
|
||||
expect(result).toHaveProperty('rentedUnitsCount');
|
||||
expect(result).toHaveProperty('staleUnitsCount');
|
||||
expect(result).toHaveProperty('errors');
|
||||
|
||||
// Verify types
|
||||
expect(typeof result.duration).toBe('number');
|
||||
expect(result.duration).toBeGreaterThanOrEqual(0);
|
||||
expect(typeof result.startedAt).toBe('string');
|
||||
expect(typeof result.completedAt).toBe('string');
|
||||
expect(Array.isArray(result.errors)).toBe(true);
|
||||
});
|
||||
|
||||
it('should generate a jobId when not provided', async () => {
|
||||
const html = createSampleHtml(['UNIT-GEN']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.jobId).toBeTruthy();
|
||||
expect(typeof result.jobId).toBe('string');
|
||||
// UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
|
||||
expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: dryRun option
|
||||
// ============================================================
|
||||
it('should skip DB writes when dryRun is true', async () => {
|
||||
const html = createSampleHtml(['DRY-A', 'DRY-B']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-dry-run',
|
||||
htmlContent: html,
|
||||
dryRun: true
|
||||
});
|
||||
|
||||
expect(result.status).toBe('success');
|
||||
expect(result.dryRun).toBe(true);
|
||||
expect(result.unitsProcessed).toBe(2);
|
||||
expect(result.pricesInserted).toBe(0);
|
||||
expect(result.staleUnitsCount).toBe(0);
|
||||
|
||||
// Verify no units were written to the units collection
|
||||
const unitsCount = await db.collection('units_scraper').countDocuments();
|
||||
expect(unitsCount).toBe(0);
|
||||
|
||||
// Verify no prices were written
|
||||
const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
|
||||
expect(pricesCount).toBe(0);
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: htmlContent option
|
||||
// ============================================================
|
||||
it('should use provided HTML instead of fetching when htmlContent is given', async () => {
|
||||
const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
|
||||
|
||||
// Mock axios.get to track if HTTP fetch is attempted
|
||||
const axios = require('axios');
|
||||
const axiosSpy = jest.spyOn(axios, 'get');
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-html-content',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.status).toBe('success');
|
||||
expect(result.unitsProcessed).toBe(3);
|
||||
// axios.get should NOT have been called since we provided htmlContent
|
||||
expect(axiosSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Records scraper run on success
|
||||
// ============================================================
|
||||
it('should record scraper run to history on success', async () => {
|
||||
const html = createSampleHtml(['REC-A']);
|
||||
|
||||
await runScrape(db, {
|
||||
jobId: 'test-record-success',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
|
||||
expect(runRecord).toBeTruthy();
|
||||
expect(runRecord.status).toBe('success');
|
||||
expect(typeof runRecord.recordedAt).toBe('string');
|
||||
expect(runRecord.unitsProcessed).toBe(1);
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Records scraper run on failure
|
||||
// ============================================================
|
||||
it('should record scraper run to history on failure', async () => {
|
||||
const html = createSampleHtml(['FAIL-A']);
|
||||
|
||||
// Create a db proxy that throws on bulkWrite (used by upsertUnits)
|
||||
// but allows other operations (like scraper_runs insertOne) to pass through
|
||||
const faultyDb = {
|
||||
collection: (name) => {
|
||||
const realCollection = db.collection(name);
|
||||
if (name === 'units_scraper') {
|
||||
return new Proxy(realCollection, {
|
||||
get(target, prop) {
|
||||
if (prop === 'bulkWrite') {
|
||||
return async () => {
|
||||
throw new Error('Database connection lost');
|
||||
};
|
||||
}
|
||||
const value = target[prop];
|
||||
if (typeof value === 'function') {
|
||||
return value.bind(target);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
});
|
||||
}
|
||||
return realCollection;
|
||||
}
|
||||
};
|
||||
|
||||
const result = await runScrape(faultyDb, {
|
||||
jobId: 'test-record-failure',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.errors).toContain('Database connection lost');
|
||||
|
||||
// The run should still be recorded (via the real db passed through for scraper_runs)
|
||||
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
|
||||
expect(runRecord).toBeTruthy();
|
||||
expect(runRecord.status).toBe('failed');
|
||||
expect(runRecord.errors).toContain('Database connection lost');
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Catches errors from fetchPage
|
||||
// ============================================================
|
||||
it('should catch and handle errors from fetchPage', async () => {
|
||||
// Mock axios.get to throw a network error (fetchPage uses axios internally)
|
||||
const axios = require('axios');
|
||||
jest.spyOn(axios, 'get').mockRejectedValue(
|
||||
new Error('Network timeout')
|
||||
);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-fetch-error'
|
||||
// No htmlContent, so it will call fetchPage which uses axios
|
||||
});
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.errors).toContain('Network timeout');
|
||||
expect(result.unitsProcessed).toBe(0);
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Catches errors from database operations
|
||||
// ============================================================
|
||||
it('should catch and handle errors from database operations', async () => {
|
||||
const html = createSampleHtml(['DB-ERR']);
|
||||
|
||||
// Create a db proxy that throws on the prices collection bulkWrite
|
||||
// Use Proxy to properly delegate all methods to the real collection
|
||||
const faultyDb = {
|
||||
collection: (name) => {
|
||||
const realCollection = db.collection(name);
|
||||
if (name === 'unit_prices_scraper') {
|
||||
return new Proxy(realCollection, {
|
||||
get(target, prop) {
|
||||
if (prop === 'bulkWrite') {
|
||||
return async () => {
|
||||
throw new Error('Write concern timeout');
|
||||
};
|
||||
}
|
||||
const value = target[prop];
|
||||
if (typeof value === 'function') {
|
||||
return value.bind(target);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
});
|
||||
}
|
||||
return realCollection;
|
||||
}
|
||||
};
|
||||
|
||||
const result = await runScrape(faultyDb, {
|
||||
jobId: 'test-db-error',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.errors).toContain('Write concern timeout');
|
||||
|
||||
// Run should still be recorded despite error
|
||||
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
|
||||
expect(runRecord).toBeTruthy();
|
||||
expect(runRecord.status).toBe('failed');
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Calculates newUnitsCount and rentedUnitsCount correctly
|
||||
// ============================================================
|
||||
it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
|
||||
// First, simulate yesterday's data by inserting price records for yesterday
|
||||
const today = new Date();
|
||||
const yesterday = new Date(today);
|
||||
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
||||
const yesterdayStr = yesterday.toISOString().split('T')[0];
|
||||
|
||||
// Yesterday had units: OLD-A, OLD-B, OLD-C
|
||||
await db.collection('unit_prices_scraper').insertMany([
|
||||
{ unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
|
||||
{ unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
|
||||
{ unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
|
||||
]);
|
||||
|
||||
// Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
|
||||
const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-calc-changes',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.status).toBe('success');
|
||||
// NEW-D is new (not in yesterday's data)
|
||||
expect(result.newUnitsCount).toBe(1);
|
||||
// OLD-C was in yesterday's data but not in today's scrape
|
||||
expect(result.rentedUnitsCount).toBe(1);
|
||||
expect(result.unitsProcessed).toBe(3);
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Default trigger is 'manual'
|
||||
// ============================================================
|
||||
it('should default trigger to manual', async () => {
|
||||
const html = createSampleHtml(['DEF-A']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-default-trigger',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.trigger).toBe('manual');
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Supports scheduled trigger
|
||||
// ============================================================
|
||||
it('should support scheduled trigger', async () => {
|
||||
const html = createSampleHtml(['SCHED-A']);
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-scheduled-trigger',
|
||||
trigger: 'scheduled',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.trigger).toBe('scheduled');
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: Handles empty HTML (no units found)
|
||||
// ============================================================
|
||||
it('should handle HTML with no units gracefully', async () => {
|
||||
const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
|
||||
|
||||
const result = await runScrape(db, {
|
||||
jobId: 'test-empty-html',
|
||||
htmlContent: emptyHtml
|
||||
});
|
||||
|
||||
expect(result.status).toBe('success');
|
||||
expect(result.unitsProcessed).toBe(0);
|
||||
expect(result.errors).toContain('No units found in HTML');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: sanitizeError - Error message sanitization
|
||||
// ============================================================
|
||||
describe('sanitizeError', () => {
|
||||
let sanitizeError;
|
||||
|
||||
beforeAll(() => {
|
||||
({ sanitizeError } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// File path sanitization
|
||||
// ----------------------------------------------------------
|
||||
describe('file path sanitization', () => {
|
||||
it('should remove Unix absolute file paths from error messages', () => {
|
||||
const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toMatch(/\/home\/user/);
|
||||
expect(sanitized.message).toContain('ENOENT');
|
||||
});
|
||||
|
||||
it('should remove /var paths from error messages', () => {
|
||||
const error = new Error('Failed to read /var/app/current/data/secrets.yml');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toMatch(/\/var\/app/);
|
||||
expect(sanitized.message).toContain('Failed to read');
|
||||
});
|
||||
|
||||
it('should remove Windows-style file paths from error messages', () => {
|
||||
const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toMatch(/C:\\Users/);
|
||||
expect(sanitized.message).toContain('Cannot find module');
|
||||
});
|
||||
|
||||
it('should remove /tmp and /usr paths from error messages', () => {
|
||||
const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toMatch(/\/tmp\//);
|
||||
expect(sanitized.message).not.toMatch(/\/usr\//);
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// MongoDB connection string sanitization
|
||||
// ----------------------------------------------------------
|
||||
describe('connection string sanitization', () => {
|
||||
it('should redact mongodb:// connection strings', () => {
|
||||
const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toContain('s3cretP4ss');
|
||||
expect(sanitized.message).not.toContain('admin:');
|
||||
expect(sanitized.message).toContain('Connection failed');
|
||||
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
|
||||
});
|
||||
|
||||
it('should redact mongodb+srv:// connection strings', () => {
|
||||
const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toContain('password123');
|
||||
expect(sanitized.message).not.toContain('user:');
|
||||
expect(sanitized.message).toContain('Timeout connecting to');
|
||||
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
|
||||
});
|
||||
|
||||
it('should redact connection string without credentials', () => {
|
||||
const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
|
||||
expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// Credential / secret sanitization
|
||||
// ----------------------------------------------------------
|
||||
describe('credential sanitization', () => {
|
||||
it('should redact common environment variable patterns', () => {
|
||||
const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toContain('sk-abc123xyz');
|
||||
expect(sanitized.message).not.toContain('bearer-9876');
|
||||
});
|
||||
|
||||
it('should redact password patterns', () => {
|
||||
const error = new Error('Auth failed with password=MyS3cret!');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).not.toContain('MyS3cret!');
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// Error type preservation
|
||||
// ----------------------------------------------------------
|
||||
describe('error type preservation', () => {
|
||||
it('should preserve the error type/name', () => {
|
||||
const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.name).toBe('TypeError');
|
||||
});
|
||||
|
||||
it('should preserve custom error names', () => {
|
||||
const error = new Error('Timeout at /var/app/scraper.js:100');
|
||||
error.name = 'TimeoutError';
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.name).toBe('TimeoutError');
|
||||
});
|
||||
|
||||
it('should preserve error name for RangeError', () => {
|
||||
const error = new RangeError('Maximum call stack size exceeded');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.name).toBe('RangeError');
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// General description preserved for debugging
|
||||
// ----------------------------------------------------------
|
||||
describe('general description preservation', () => {
|
||||
it('should preserve a useful general description', () => {
|
||||
const error = new Error('Network timeout after 30000ms');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).toContain('Network timeout after 30000ms');
|
||||
});
|
||||
|
||||
it('should preserve error description when no sensitive data present', () => {
|
||||
const error = new Error('Request failed with status code 500');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).toBe('Request failed with status code 500');
|
||||
});
|
||||
|
||||
it('should return a useful message even after heavy sanitization', () => {
|
||||
const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.message).toContain('ECONNREFUSED');
|
||||
expect(sanitized.message.length).toBeGreaterThan(5);
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// Stack trace sanitization
|
||||
// ----------------------------------------------------------
|
||||
describe('stack trace removal', () => {
|
||||
it('should remove file paths from stack traces', () => {
|
||||
const error = new Error('Something failed');
|
||||
error.stack = 'Error: Something failed\n at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.stack).not.toMatch(/\/home\/user/);
|
||||
expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
|
||||
});
|
||||
|
||||
it('should handle errors without stack trace', () => {
|
||||
const error = new Error('No stack');
|
||||
error.stack = undefined;
|
||||
const sanitized = sanitizeError(error);
|
||||
expect(sanitized.stack).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// Integration: sanitization applied before recordScraperRun()
|
||||
// ----------------------------------------------------------
|
||||
describe('sanitization before recordScraperRun()', () => {
|
||||
it('should store sanitized error message in scraper_runs on failure', async () => {
|
||||
const html = createSampleHtml(['SANITIZE-A']);
|
||||
|
||||
// Create a db proxy that throws an error containing sensitive info
|
||||
const sensitiveError = new Error(
|
||||
'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
|
||||
);
|
||||
sensitiveError.name = 'MongoServerError';
|
||||
|
||||
const faultyDb = {
|
||||
collection: (name) => {
|
||||
const realCollection = db.collection(name);
|
||||
if (name === 'units_scraper') {
|
||||
return new Proxy(realCollection, {
|
||||
get(target, prop) {
|
||||
if (prop === 'bulkWrite') {
|
||||
return async () => { throw sensitiveError; };
|
||||
}
|
||||
const value = target[prop];
|
||||
if (typeof value === 'function') {
|
||||
return value.bind(target);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
});
|
||||
}
|
||||
return realCollection;
|
||||
}
|
||||
};
|
||||
|
||||
const result = await runScrape(faultyDb, {
|
||||
jobId: 'test-sanitized-error',
|
||||
htmlContent: html
|
||||
});
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
|
||||
// Verify the error stored in result.errors is sanitized
|
||||
const errorMsg = result.errors[0];
|
||||
expect(errorMsg).not.toContain('SuperSecret');
|
||||
expect(errorMsg).not.toContain('admin:');
|
||||
expect(errorMsg).not.toContain('/home/deploy/');
|
||||
expect(errorMsg).toContain('MongoServerError');
|
||||
|
||||
// Verify the error stored in scraper_runs is sanitized
|
||||
const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
|
||||
expect(runRecord).toBeTruthy();
|
||||
expect(runRecord.status).toBe('failed');
|
||||
const storedError = runRecord.errors[0];
|
||||
expect(storedError).not.toContain('SuperSecret');
|
||||
expect(storedError).not.toContain('admin:');
|
||||
expect(storedError).not.toContain('/home/deploy/');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Test: createScraperIndexes
|
||||
// ============================================================
|
||||
describe('createScraperIndexes', () => {
|
||||
const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger.info.mockClear();
|
||||
mockLogger.warn.mockClear();
|
||||
mockLogger.error.mockClear();
|
||||
});
|
||||
|
||||
it('should create the status_startedAt compound index', async () => {
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
|
||||
expect(statusIndex).toBeTruthy();
|
||||
expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
|
||||
});
|
||||
|
||||
it('should create the startedAt_desc index', async () => {
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
|
||||
expect(startedAtIndex).toBeTruthy();
|
||||
expect(startedAtIndex.key).toEqual({ startedAt: -1 });
|
||||
});
|
||||
|
||||
it('should log success after creating indexes', async () => {
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
|
||||
});
|
||||
|
||||
it('should be idempotent (no error on second call)', async () => {
|
||||
await createScraperIndexes(db, mockLogger);
|
||||
|
||||
// Calling again should not throw
|
||||
await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
|
||||
|
||||
// Indexes should still exist
|
||||
const indexes = await db.collection('scraper_runs').indexes();
|
||||
const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
|
||||
const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
|
||||
expect(statusIndex).toBeTruthy();
|
||||
expect(startedAtIndex).toBeTruthy();
|
||||
});
|
||||
|
||||
it('should handle database errors gracefully', async () => {
|
||||
// Pass an object that will throw when collection() is called
|
||||
const faultyDb = {
|
||||
collection: () => {
|
||||
throw new Error('Connection lost');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
});
|
||||
@ -1,177 +0,0 @@
|
||||
/**
|
||||
* Tests for scraper integration in server.js startup
|
||||
*
|
||||
* Covers:
|
||||
* - Server initializes scraper scheduler on startup
|
||||
* - Server does not crash if scheduler init fails
|
||||
* - Indexes are created before scheduler starts
|
||||
* - Scheduler not initialized when SCRAPER_ENABLED=false
|
||||
*
|
||||
* Strategy: Instead of importing server.js directly (which starts Express and
|
||||
* connects to MongoDB), we replicate the scraper initialization logic from
|
||||
* connectToMongoDB() and test it with mocked dependencies. This verifies the
|
||||
* integration contract without needing a full server boot.
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
|
||||
describe('Server Scraper Integration', () => {
|
||||
let connection;
|
||||
let db;
|
||||
|
||||
beforeAll(async () => {
|
||||
const uri = process.env.MONGO_URI;
|
||||
connection = await MongoClient.connect(uri);
|
||||
db = connection.db('apartments_server_integration_test');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (connection) {
|
||||
await connection.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe('scraper initialization during startup', () => {
|
||||
let mockCreateScraperIndexes;
|
||||
let mockInitializeScheduler;
|
||||
let mockCreateLogger;
|
||||
let mockLogger;
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger = {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
debug: jest.fn()
|
||||
};
|
||||
mockCreateLogger = jest.fn(() => mockLogger);
|
||||
mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
|
||||
mockInitializeScheduler = jest.fn();
|
||||
});
|
||||
|
||||
/**
|
||||
* Replicates the scraper initialization logic from server.js connectToMongoDB().
|
||||
* This is the exact pattern used in production:
|
||||
*
|
||||
* try {
|
||||
* const scraperLogger = createLogger('server-init');
|
||||
* await createScraperIndexes(db, scraperLogger);
|
||||
* if (scraperConfig.SCRAPER_ENABLED) {
|
||||
* initializeScheduler(db);
|
||||
* }
|
||||
* } catch (error) {
|
||||
* console.error('Scraper initialization failed (server continues):', error.message);
|
||||
* }
|
||||
*/
|
||||
async function runScraperInit(testDb, config) {
|
||||
try {
|
||||
const scraperLogger = mockCreateLogger('server-init');
|
||||
await mockCreateScraperIndexes(testDb, scraperLogger);
|
||||
if (config.SCRAPER_ENABLED) {
|
||||
mockInitializeScheduler(testDb);
|
||||
}
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
it('should initialize scraper scheduler on startup when enabled', async () => {
|
||||
const config = { SCRAPER_ENABLED: true };
|
||||
|
||||
const result = await runScraperInit(db, config);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
|
||||
expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
|
||||
});
|
||||
|
||||
it('should not crash if createScraperIndexes throws an error', async () => {
|
||||
mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
|
||||
const config = { SCRAPER_ENABLED: true };
|
||||
|
||||
const result = await runScraperInit(db, config);
|
||||
|
||||
// The server should continue (error is caught, not thrown)
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toBe('Index creation failed');
|
||||
// initializeScheduler should NOT have been called since the error occurred before it
|
||||
expect(mockInitializeScheduler).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should create indexes before initializing the scheduler', async () => {
|
||||
const callOrder = [];
|
||||
mockCreateScraperIndexes.mockImplementation(async () => {
|
||||
callOrder.push('createScraperIndexes');
|
||||
});
|
||||
mockInitializeScheduler.mockImplementation(() => {
|
||||
callOrder.push('initializeScheduler');
|
||||
});
|
||||
|
||||
const config = { SCRAPER_ENABLED: true };
|
||||
await runScraperInit(db, config);
|
||||
|
||||
expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
|
||||
});
|
||||
|
||||
it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
|
||||
const config = { SCRAPER_ENABLED: false };
|
||||
|
||||
const result = await runScraperInit(db, config);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
|
||||
expect(mockInitializeScheduler).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('server.js imports and exports verification', () => {
|
||||
it('should be able to import createScraperIndexes from scraperService', () => {
|
||||
const { createScraperIndexes } = require('../../services/scraperService');
|
||||
expect(typeof createScraperIndexes).toBe('function');
|
||||
});
|
||||
|
||||
it('should be able to import initializeScheduler from scraperJob', () => {
|
||||
const { initializeScheduler } = require('../../jobs/scraperJob');
|
||||
expect(typeof initializeScheduler).toBe('function');
|
||||
});
|
||||
|
||||
it('should be able to import registerSignalHandlers from scraperJob', () => {
|
||||
const { registerSignalHandlers } = require('../../jobs/scraperJob');
|
||||
expect(typeof registerSignalHandlers).toBe('function');
|
||||
});
|
||||
|
||||
it('should be able to import SCRAPER_ENABLED from scraper config', () => {
|
||||
const scraperConfig = require('../../config/scraper');
|
||||
expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createScraperIndexes error handling with real db', () => {
|
||||
it('should handle index creation gracefully with a valid db', async () => {
|
||||
const { createScraperIndexes } = require('../../services/scraperService');
|
||||
const { createLogger } = require('../../services/scraperLogger');
|
||||
const logger = createLogger('test-server-init');
|
||||
|
||||
// Should not throw - creates indexes on the test db
|
||||
await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw when given an invalid db object', async () => {
|
||||
const { createScraperIndexes } = require('../../services/scraperService');
|
||||
const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
|
||||
|
||||
// Passing null should throw (simulates what the try/catch in server.js protects against)
|
||||
await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('registerSignalHandlers', () => {
|
||||
it('should register SIGTERM and SIGINT handlers without throwing', () => {
|
||||
const { registerSignalHandlers } = require('../../jobs/scraperJob');
|
||||
|
||||
// Should not throw when called
|
||||
expect(() => registerSignalHandlers()).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,561 +0,0 @@
|
||||
/**
|
||||
* Tests for updateDailySummary() daily summary aggregation
|
||||
*
|
||||
* Covers:
|
||||
* - Upsert by date key (YYYY-MM-DD)
|
||||
* - Summary includes new_units, rented_units, stale_units arrays
|
||||
* - Count fields are calculated correctly
|
||||
* - net_change = newUnits.length - rentedUnits.length
|
||||
* - total_available_today is set correctly
|
||||
* - total_available_yesterday fetched from previous day
|
||||
* - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
|
||||
* - turnover_rate is 0 when yesterdayTotal is 0
|
||||
* - Error handling when updateOne fails
|
||||
* - Logging of summary update details
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require updateDailySummary after implementation
|
||||
let updateDailySummary;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ updateDailySummary } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('updateDailySummary', () => {
|
||||
// ---------------------------------------------------------------
|
||||
// 1. Upsert by date key
|
||||
// ---------------------------------------------------------------
|
||||
describe('upsert by date key', () => {
|
||||
it('should insert a new summary when no document exists for the date', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 2,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
const result = await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
// Should have upserted (created new)
|
||||
expect(result.upsertedCount).toBe(1);
|
||||
|
||||
// Verify the document in the database
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc).not.toBeNull();
|
||||
expect(doc.date).toBe('2026-02-05');
|
||||
});
|
||||
|
||||
it('should update an existing summary for the same date (idempotent upsert)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert initial summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-05',
|
||||
new_units: ['A101'],
|
||||
rented_units: [],
|
||||
total_available_today: 10
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 1,
|
||||
totalAvailable: 12
|
||||
};
|
||||
|
||||
const result = await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
// Should have matched and modified (not upserted)
|
||||
expect(result.matchedCount).toBe(1);
|
||||
|
||||
// Verify updated values
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
|
||||
expect(doc.total_available_today).toBe(12);
|
||||
|
||||
// Verify only one document exists for this date
|
||||
const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. Summary includes new_units, rented_units, stale_units arrays
|
||||
// ---------------------------------------------------------------
|
||||
describe('summary arrays', () => {
|
||||
it('should store new_units, rented_units, and stale_units arrays', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201', 'B202', 'B203'],
|
||||
staleUnitsCount: 3,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual(['A101', 'A102']);
|
||||
expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
|
||||
expect(doc.stale_units).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle empty arrays for new_units and rented_units', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 25
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual([]);
|
||||
expect(doc.rented_units).toEqual([]);
|
||||
expect(doc.stale_units).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Count fields are calculated correctly
|
||||
// ---------------------------------------------------------------
|
||||
describe('count fields', () => {
|
||||
it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 5,
|
||||
totalAvailable: 30
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units_count).toBe(3);
|
||||
expect(doc.rented_units_count).toBe(1);
|
||||
expect(doc.stale_units_count).toBe(5);
|
||||
});
|
||||
|
||||
it('should set all count fields to 0 when arrays are empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units_count).toBe(0);
|
||||
expect(doc.rented_units_count).toBe(0);
|
||||
expect(doc.stale_units_count).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. net_change = newUnits.length - rentedUnits.length
|
||||
// ---------------------------------------------------------------
|
||||
describe('net_change calculation', () => {
|
||||
it('should calculate positive net_change when more units added than rented', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(2); // 3 - 1 = 2
|
||||
});
|
||||
|
||||
it('should calculate negative net_change when more units rented than added', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201', 'B202', 'B203', 'B204'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(-3); // 1 - 4 = -3
|
||||
});
|
||||
|
||||
it('should calculate zero net_change when equal new and rented', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201', 'B202'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. total_available_today is set correctly
|
||||
// ---------------------------------------------------------------
|
||||
describe('total_available_today', () => {
|
||||
it('should store the totalAvailable value as total_available_today', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 42
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_today).toBe(42);
|
||||
});
|
||||
|
||||
it('should handle totalAvailable of 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 0
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_today).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. total_available_yesterday fetched from previous day
|
||||
// ---------------------------------------------------------------
|
||||
describe('total_available_yesterday', () => {
|
||||
it('should fetch total_available_today from previous day summary', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 35,
|
||||
new_units: [],
|
||||
rented_units: []
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 35
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_yesterday).toBe(35);
|
||||
});
|
||||
|
||||
it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_yesterday).toBe(0);
|
||||
});
|
||||
|
||||
it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert March 31 summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-03-31',
|
||||
total_available_today: 50
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-04-01',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 48
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
|
||||
expect(doc.total_available_yesterday).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
|
||||
// ---------------------------------------------------------------
|
||||
describe('turnover_rate calculation', () => {
|
||||
it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 50 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 50
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 46
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
// 5 rented / 50 yesterday = 10%
|
||||
expect(doc.turnover_rate).toBe(10);
|
||||
});
|
||||
|
||||
it('should round turnover_rate to 2 decimal places', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 30 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 30
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: ['B201', 'B202'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 28
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
// 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
|
||||
expect(doc.turnover_rate).toBe(6.67);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. turnover_rate is 0 when yesterdayTotal is 0
|
||||
// ---------------------------------------------------------------
|
||||
describe('turnover_rate edge cases', () => {
|
||||
it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
|
||||
it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 0 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 0
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
|
||||
it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 9. Error handling when updateOne fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw and log error when updateOne fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
// Create a mock db that throws on updateOne
|
||||
const mockCollection = {
|
||||
findOne: jest.fn().mockResolvedValue(null),
|
||||
updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(
|
||||
updateDailySummary(mockDb, summaryData, logger)
|
||||
).rejects.toThrow('Write concern timeout');
|
||||
|
||||
// Verify error was logged
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to update daily summary',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 10. Logging of summary update details
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log summary update details on success', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 1,
|
||||
totalAvailable: 18
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Daily summary updated',
|
||||
expect.objectContaining({
|
||||
date: '2026-02-05',
|
||||
newUnits: 2,
|
||||
rentedUnits: 1,
|
||||
totalAvailable: 18
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should include timestamp field in the stored summary', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.timestamp).toBeDefined();
|
||||
expect(typeof doc.timestamp).toBe('string');
|
||||
// Verify it matches the Denver-local timestamp format (YYYY-MM-DDTHH:mm:ss.mmm)
|
||||
expect(doc.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}$/);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,499 +0,0 @@
|
||||
/**
|
||||
* Tests for upsertUnits() bulk operation
|
||||
*
|
||||
* Covers:
|
||||
* - bulkWrite is called with correct updateOne operations
|
||||
* - upsert: true is set for all operations
|
||||
* - $set updates last_scraped and data_source fields
|
||||
* - $setOnInsert sets first_seen for new units
|
||||
* - Empty units array logs warning and returns early
|
||||
* - ordered: false is set for parallel execution
|
||||
* - Error handling when bulkWrite fails
|
||||
* - Logging of matched, modified, and upserted counts
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// We will require upsertUnits and getNow after implementation
|
||||
let upsertUnits;
|
||||
let getNow;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ upsertUnits, getNow } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean the units collection before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('upsertUnits', () => {
|
||||
const UNITS_COLLECTION = 'units_scraper';
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. bulkWrite is called with correct updateOne operations
|
||||
// ---------------------------------------------------------------
|
||||
describe('bulkWrite operations', () => {
|
||||
it('should create updateOne operations for each unit', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
|
||||
{ unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
|
||||
];
|
||||
|
||||
const result = await upsertUnits(db, units, logger);
|
||||
|
||||
// Both units should be upserted (new inserts)
|
||||
expect(result.upsertedCount).toBe(2);
|
||||
|
||||
// Verify documents exist in the collection
|
||||
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(2);
|
||||
|
||||
const unitCodes = docs.map(d => d.unit_code).sort();
|
||||
expect(unitCodes).toEqual(['A101', 'B202']);
|
||||
});
|
||||
|
||||
it('should filter by unit_code as the unique key', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert a unit first
|
||||
const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
// Upsert again with updated price
|
||||
const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
|
||||
const result = await upsertUnits(db, updatedUnits, logger);
|
||||
|
||||
// Should match and modify, not insert new
|
||||
expect(result.matchedCount).toBe(1);
|
||||
expect(result.upsertedCount).toBe(0);
|
||||
|
||||
// Verify only one document exists
|
||||
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
expect(docs[0].price).toBe(1400);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. upsert: true is set for all operations
|
||||
// ---------------------------------------------------------------
|
||||
describe('upsert behavior', () => {
|
||||
it('should insert new units when they do not exist (upsert: true)', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'NEW-001', price: 1100 },
|
||||
{ unit_code: 'NEW-002', price: 1200 },
|
||||
{ unit_code: 'NEW-003', price: 1300 }
|
||||
];
|
||||
|
||||
const result = await upsertUnits(db, units, logger);
|
||||
|
||||
expect(result.upsertedCount).toBe(3);
|
||||
|
||||
// Verify all three are in the database
|
||||
const count = await db.collection(UNITS_COLLECTION).countDocuments();
|
||||
expect(count).toBe(3);
|
||||
});
|
||||
|
||||
it('should update existing units without creating duplicates', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First insert
|
||||
await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
|
||||
|
||||
// Second upsert with same unit_code
|
||||
await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
|
||||
|
||||
const count = await db.collection(UNITS_COLLECTION).countDocuments();
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. $set updates last_scraped and data_source fields
|
||||
// ---------------------------------------------------------------
|
||||
describe('$set fields', () => {
|
||||
it('should set last_scraped timestamp on every upsert', async () => {
|
||||
const logger = createMockLogger();
|
||||
const beforeTime = getNow();
|
||||
|
||||
const units = [{ unit_code: 'T100', price: 1000 }];
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
const afterTime = getNow();
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
|
||||
expect(doc.last_scraped).toBeDefined();
|
||||
expect(typeof doc.last_scraped).toBe('string');
|
||||
// The last_scraped should be between before and after timestamps
|
||||
expect(doc.last_scraped >= beforeTime).toBe(true);
|
||||
expect(doc.last_scraped <= afterTime).toBe(true);
|
||||
});
|
||||
|
||||
it('should set data_source to "web_scraper"', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'T200', price: 1100 }];
|
||||
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
|
||||
expect(doc.data_source).toBe('web_scraper');
|
||||
});
|
||||
|
||||
it('should update last_scraped on subsequent upserts', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First insert
|
||||
await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
|
||||
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
|
||||
const firstScraped = doc1.last_scraped;
|
||||
|
||||
// Small delay to ensure different timestamp
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
// Second upsert
|
||||
await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
|
||||
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
|
||||
const secondScraped = doc2.last_scraped;
|
||||
|
||||
expect(secondScraped > firstScraped).toBe(true);
|
||||
});
|
||||
|
||||
it('should spread all unit fields into the $set operation', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{
|
||||
unit_code: 'T400',
|
||||
price: 1500,
|
||||
bed_count: 2,
|
||||
bath_count: 1.5,
|
||||
area: 900,
|
||||
floor: 3,
|
||||
plan_name: 'Luxury',
|
||||
community: 'Tower A'
|
||||
}];
|
||||
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
|
||||
expect(doc.price).toBe(1500);
|
||||
expect(doc.bed_count).toBe(2);
|
||||
expect(doc.bath_count).toBe(1.5);
|
||||
expect(doc.area).toBe(900);
|
||||
expect(doc.floor).toBe(3);
|
||||
expect(doc.plan_name).toBe('Luxury');
|
||||
expect(doc.community).toBe('Tower A');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. $setOnInsert sets first_seen for new units
|
||||
// ---------------------------------------------------------------
|
||||
describe('$setOnInsert for first_seen', () => {
|
||||
it('should set first_seen on initial insert', async () => {
|
||||
const logger = createMockLogger();
|
||||
const beforeTime = getNow();
|
||||
|
||||
await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
|
||||
expect(doc.first_seen).toBeDefined();
|
||||
expect(typeof doc.first_seen).toBe('string');
|
||||
expect(doc.first_seen >= beforeTime).toBe(true);
|
||||
});
|
||||
|
||||
it('should NOT update first_seen on subsequent upserts', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First insert
|
||||
await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
|
||||
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
|
||||
const originalFirstSeen = doc1.first_seen;
|
||||
|
||||
// Small delay to ensure different timestamp
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
// Second upsert (update)
|
||||
await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
|
||||
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
|
||||
|
||||
// first_seen should remain unchanged
|
||||
expect(doc2.first_seen).toBe(originalFirstSeen);
|
||||
});
|
||||
|
||||
it('should set different first_seen for different units inserted at different times', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
|
||||
const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
|
||||
const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
|
||||
|
||||
// Both should have first_seen, but different values
|
||||
expect(doc1.first_seen).toBeDefined();
|
||||
expect(doc2.first_seen).toBeDefined();
|
||||
expect(doc2.first_seen > doc1.first_seen).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Empty units array logs warning and returns early
|
||||
// ---------------------------------------------------------------
|
||||
describe('empty units array', () => {
|
||||
it('should log a warning when units array is empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await upsertUnits(db, [], logger);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
|
||||
});
|
||||
|
||||
it('should return early with zero counts for empty array', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const result = await upsertUnits(db, [], logger);
|
||||
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
expect(result.upsertedCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should not perform any database writes for empty array', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await upsertUnits(db, [], logger);
|
||||
|
||||
const count = await db.collection(UNITS_COLLECTION).countDocuments();
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. ordered: false is set for parallel execution
|
||||
// ---------------------------------------------------------------
|
||||
describe('ordered: false for parallel execution', () => {
|
||||
it('should successfully process all units even if one has a conflict', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert multiple units - ordered: false means all operations execute
|
||||
// even if some fail. We verify by checking all valid units are present.
|
||||
const units = [
|
||||
{ unit_code: 'P100', price: 1000 },
|
||||
{ unit_code: 'P200', price: 1100 },
|
||||
{ unit_code: 'P300', price: 1200 },
|
||||
{ unit_code: 'P400', price: 1300 },
|
||||
{ unit_code: 'P500', price: 1400 }
|
||||
];
|
||||
|
||||
const result = await upsertUnits(db, units, logger);
|
||||
|
||||
expect(result.upsertedCount).toBe(5);
|
||||
|
||||
// All 5 units should be in the database
|
||||
const count = await db.collection(UNITS_COLLECTION).countDocuments();
|
||||
expect(count).toBe(5);
|
||||
});
|
||||
|
||||
it('should handle a mix of new inserts and existing updates', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First, insert some units
|
||||
await upsertUnits(db, [
|
||||
{ unit_code: 'M100', price: 900 },
|
||||
{ unit_code: 'M200', price: 1000 }
|
||||
], logger);
|
||||
|
||||
// Now upsert a mix of existing and new units
|
||||
const mixedUnits = [
|
||||
{ unit_code: 'M100', price: 950 }, // existing - update
|
||||
{ unit_code: 'M200', price: 1050 }, // existing - update
|
||||
{ unit_code: 'M300', price: 1100 }, // new - insert
|
||||
];
|
||||
|
||||
const result = await upsertUnits(db, mixedUnits, logger);
|
||||
|
||||
expect(result.matchedCount).toBe(2);
|
||||
expect(result.upsertedCount).toBe(1);
|
||||
|
||||
// Total should be 3 documents
|
||||
const count = await db.collection(UNITS_COLLECTION).countDocuments();
|
||||
expect(count).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Error handling when bulkWrite fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw error when bulkWrite fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'E100', price: 1000 }];
|
||||
|
||||
// Create a mock db that throws on bulkWrite
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
|
||||
it('should log error details when bulkWrite fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'E200', price: 1000 }];
|
||||
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
try {
|
||||
await upsertUnits(mockDb, units, logger);
|
||||
} catch (e) {
|
||||
// Expected to throw
|
||||
}
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to upsert units',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should re-throw the original error', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'E300', price: 1000 }];
|
||||
|
||||
const originalError = new TypeError('Invalid operation');
|
||||
const mockCollection = {
|
||||
bulkWrite: jest.fn().mockRejectedValue(originalError)
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. Logging of matched, modified, and upserted counts
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging of result counts', () => {
|
||||
it('should log matched, modified, and upserted counts on success', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'L100', price: 1000 },
|
||||
{ unit_code: 'L200', price: 1100 }
|
||||
];
|
||||
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Units upserted',
|
||||
expect.objectContaining({
|
||||
matched: expect.any(Number),
|
||||
modified: expect.any(Number),
|
||||
upserted: expect.any(Number)
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log correct counts for new inserts', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [
|
||||
{ unit_code: 'L300', price: 1000 },
|
||||
{ unit_code: 'L400', price: 1100 }
|
||||
];
|
||||
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
// For new inserts: matched=0, modified=0, upserted=2
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Units upserted',
|
||||
expect.objectContaining({
|
||||
matched: 0,
|
||||
modified: 0,
|
||||
upserted: 2
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should log correct counts for updates to existing units', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// First insert
|
||||
await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
|
||||
|
||||
// Reset mock to capture only the second call
|
||||
logger.info.mockClear();
|
||||
|
||||
// Update existing
|
||||
await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Units upserted',
|
||||
expect.objectContaining({
|
||||
matched: 1,
|
||||
modified: 1,
|
||||
upserted: 0
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 9. Return value
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value', () => {
|
||||
it('should return the bulkWrite result object', async () => {
|
||||
const logger = createMockLogger();
|
||||
const units = [{ unit_code: 'R100', price: 1000 }];
|
||||
|
||||
const result = await upsertUnits(db, units, logger);
|
||||
|
||||
// bulkWrite result should have these standard properties
|
||||
expect(result).toHaveProperty('matchedCount');
|
||||
expect(result).toHaveProperty('modifiedCount');
|
||||
expect(result).toHaveProperty('upsertedCount');
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,41 +0,0 @@
|
||||
/**
|
||||
* Scraper Configuration Module
|
||||
*
|
||||
* Contains all constants and settings for the apartment price scraper.
|
||||
* All values can be overridden via environment variables for deployment flexibility.
|
||||
*
|
||||
* See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
// Target URL for apartment listings
|
||||
TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
|
||||
|
||||
// Scheduling configuration
|
||||
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
|
||||
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
|
||||
|
||||
// HTTP settings
|
||||
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
|
||||
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
|
||||
|
||||
// Retry configuration for HTTP requests
|
||||
RETRY_CONFIG: {
|
||||
maxRetries: 3,
|
||||
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
|
||||
timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
|
||||
},
|
||||
|
||||
// Graceful shutdown timeout (how long to wait for running job before force-stopping)
|
||||
SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
|
||||
|
||||
// MongoDB collection names - defaults to validation collections for scraper validation;
|
||||
// switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
|
||||
COLLECTIONS: {
|
||||
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper',
|
||||
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper',
|
||||
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
|
||||
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
|
||||
}
|
||||
};
|
||||
@ -1,4 +0,0 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEBOFl4SHzoZVM59+668t2tecTyKM+
|
||||
cE4zaOkbFO30u7d2+bjpwPnbYrTutKmAeehYzCAVW/OYea7ML0cW2YCSCQ==
|
||||
-----END PUBLIC KEY-----
|
||||
@ -1,8 +1,5 @@
|
||||
name: apartment-api
|
||||
|
||||
services:
|
||||
apartment-api:
|
||||
image: ${IMAGE:-apartment-api:latest}
|
||||
build: .
|
||||
container_name: apartment-api
|
||||
restart: unless-stopped
|
||||
@ -22,7 +19,9 @@ services:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.docker.network=traefik"
|
||||
- "traefik.http.routers.apartment-api.rule=Host(`apartments.maverickapplications.com`) && PathPrefix(`/api`)"
|
||||
- "traefik.http.routers.apartment-api.entrypoints=web"
|
||||
- "traefik.http.routers.apartment-api.entrypoints=websecure"
|
||||
- "traefik.http.routers.apartment-api.tls=true"
|
||||
- "traefik.http.routers.apartment-api.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.routers.apartment-api.priority=100"
|
||||
- "traefik.http.services.apartment-api.loadbalancer.server.port=8080" # Updated to match
|
||||
- "traefik.http.middlewares.apartment-api-stripprefix.stripprefix.prefixes=/api"
|
||||
|
||||
@ -1,34 +0,0 @@
|
||||
import js from '@eslint/js';
|
||||
import globals from 'globals';
|
||||
|
||||
export default [
|
||||
js.configs.recommended,
|
||||
{
|
||||
languageOptions: {
|
||||
ecmaVersion: 2022,
|
||||
sourceType: 'commonjs',
|
||||
globals: {
|
||||
...globals.node,
|
||||
...globals.jest,
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
// Allow unused vars with underscore prefix, and unused function args
|
||||
'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
|
||||
'no-console': 'off',
|
||||
// Formatting rules as warnings (won't fail CI, but will be reported)
|
||||
'semi': ['warn', 'always'],
|
||||
'no-multiple-empty-lines': ['warn', { max: 2 }],
|
||||
'eol-last': ['warn', 'always'],
|
||||
'no-trailing-spaces': 'warn',
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: [
|
||||
'node_modules/**',
|
||||
'coverage/**',
|
||||
'*.min.js',
|
||||
'eslint.config.mjs',
|
||||
],
|
||||
},
|
||||
];
|
||||
@ -1,275 +0,0 @@
|
||||
const cron = require('node-cron');
|
||||
const crypto = require('crypto');
|
||||
const config = require('../config/scraper');
|
||||
const { runScrape } = require('../services/scraperService');
|
||||
const { createLogger } = require('../services/scraperLogger');
|
||||
|
||||
// In-process mutex state
|
||||
let isRunning = false;
|
||||
let currentJobId = null;
|
||||
|
||||
// Scheduler state
|
||||
let scheduledJob = null;
|
||||
|
||||
// Shutdown state
|
||||
let shuttingDown = false;
|
||||
let runningJobPromise = null;
|
||||
|
||||
/**
|
||||
* Check if scraper is currently running
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isScraperRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current job ID if running
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function getCurrentJobId() {
|
||||
return currentJobId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the scraper lock
|
||||
* @param {string} jobId - Job ID to set
|
||||
* @returns {boolean} True if lock acquired
|
||||
*/
|
||||
function acquireLock(jobId) {
|
||||
if (isRunning || shuttingDown) {
|
||||
return false;
|
||||
}
|
||||
isRunning = true;
|
||||
currentJobId = jobId;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the scraper lock
|
||||
*/
|
||||
function releaseLock() {
|
||||
isRunning = false;
|
||||
currentJobId = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured schedule expression
|
||||
* @returns {string} Cron expression or 'disabled'
|
||||
*/
|
||||
function getScheduleExpression() {
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
return 'disabled';
|
||||
}
|
||||
return config.SCRAPER_SCHEDULE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate next scheduled run time
|
||||
* @returns {string|null} ISO timestamp or null if disabled
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
if (!config.SCRAPER_ENABLED || !scheduledJob) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { CronExpressionParser } = require('cron-parser');
|
||||
try {
|
||||
const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
|
||||
tz: config.SCRAPER_TIMEZONE
|
||||
});
|
||||
return interval.next().toISOString();
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if schedule runs more frequently than 1 hour
|
||||
* @param {string} schedule - Cron expression
|
||||
* @returns {boolean} True if schedule is too frequent
|
||||
*/
|
||||
function isScheduleTooFrequent(schedule) {
|
||||
const parts = schedule.trim().split(/\s+/);
|
||||
|
||||
if (parts.length < 5) return false;
|
||||
|
||||
const minuteField = parts[0];
|
||||
|
||||
// If minute field is */N with N < 60, it runs more than once per hour
|
||||
if (/^\*\/\d+$/.test(minuteField)) {
|
||||
const interval = parseInt(minuteField.substring(2), 10);
|
||||
if (interval < 60) return true;
|
||||
}
|
||||
|
||||
// If minute field is *, it runs every minute
|
||||
if (minuteField === '*') return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the scraper scheduler
|
||||
* @param {Db} db - MongoDB database instance
|
||||
*/
|
||||
function initializeScheduler(db) {
|
||||
const logger = createLogger('scheduler');
|
||||
|
||||
// Check if scheduling is enabled
|
||||
if (!config.SCRAPER_ENABLED) {
|
||||
logger.info('Scraper scheduling is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(config.SCRAPER_SCHEDULE)) {
|
||||
logger.error('Invalid cron schedule expression', {
|
||||
schedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
logger.warn('Falling back to default schedule: 0 6 * * *');
|
||||
config.SCRAPER_SCHEDULE = '0 6 * * *';
|
||||
}
|
||||
|
||||
// Validate minimum interval (1 hour)
|
||||
if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
|
||||
logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
|
||||
originalSchedule: config.SCRAPER_SCHEDULE
|
||||
});
|
||||
config.SCRAPER_SCHEDULE = '0 * * * *';
|
||||
}
|
||||
|
||||
// Create the scheduled job
|
||||
scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
|
||||
const jobId = crypto.randomUUID();
|
||||
const jobLogger = createLogger(jobId);
|
||||
|
||||
jobLogger.info('Scheduled scrape triggered');
|
||||
|
||||
// Check if already running or shutting down
|
||||
if (!acquireLock(jobId)) {
|
||||
jobLogger.warn('Skipped - scrape already in progress');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
|
||||
runningJobPromise = jobExecution;
|
||||
await jobExecution;
|
||||
} catch (error) {
|
||||
jobLogger.error('Scheduled scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
} finally {
|
||||
runningJobPromise = null;
|
||||
releaseLock();
|
||||
}
|
||||
}, {
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
scheduled: true
|
||||
});
|
||||
|
||||
logger.info('Scraper scheduler initialized', {
|
||||
schedule: config.SCRAPER_SCHEDULE,
|
||||
timezone: config.SCRAPER_TIMEZONE,
|
||||
nextRun: getNextScheduledRun()
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler (for graceful shutdown)
|
||||
*/
|
||||
function stopScheduler() {
|
||||
if (scheduledJob) {
|
||||
scheduledJob.stop();
|
||||
scheduledJob = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is in the process of shutting down
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isShuttingDown() {
|
||||
return shuttingDown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a graceful shutdown of the scraper
|
||||
* - Stops the cron scheduler to prevent new jobs
|
||||
* - Waits for any running job to complete (with timeout)
|
||||
* - Releases the mutex lock
|
||||
* - Logs shutdown progress
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function gracefulShutdown() {
|
||||
const logger = createLogger('shutdown');
|
||||
|
||||
logger.info('Shutdown initiated');
|
||||
|
||||
// Mark as shutting down to prevent new jobs
|
||||
shuttingDown = true;
|
||||
|
||||
// Stop the cron scheduler
|
||||
stopScheduler();
|
||||
|
||||
// Wait for running job to complete (with timeout)
|
||||
if (isRunning && runningJobPromise) {
|
||||
logger.info('Waiting for running job to complete', {
|
||||
jobId: currentJobId,
|
||||
timeout: config.SHUTDOWN_TIMEOUT
|
||||
});
|
||||
|
||||
let timeoutHandle;
|
||||
const timeoutPromise = new Promise((resolve) => {
|
||||
timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
|
||||
});
|
||||
|
||||
const result = await Promise.race([
|
||||
runningJobPromise.then(() => 'completed').catch(() => 'completed'),
|
||||
timeoutPromise
|
||||
]);
|
||||
|
||||
clearTimeout(timeoutHandle);
|
||||
|
||||
if (result === 'timeout') {
|
||||
logger.warn('Shutdown wait for running job timed out', {
|
||||
jobId: currentJobId,
|
||||
timeout: config.SHUTDOWN_TIMEOUT
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Force release the lock
|
||||
releaseLock();
|
||||
|
||||
logger.info('Shutdown complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Register process signal handlers for graceful shutdown
|
||||
* Listens for SIGTERM and SIGINT signals
|
||||
*/
|
||||
function registerSignalHandlers() {
|
||||
process.on('SIGTERM', () => {
|
||||
gracefulShutdown();
|
||||
});
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
gracefulShutdown();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isScraperRunning,
|
||||
getCurrentJobId,
|
||||
acquireLock,
|
||||
releaseLock,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun,
|
||||
initializeScheduler,
|
||||
stopScheduler,
|
||||
isShuttingDown,
|
||||
gracefulShutdown,
|
||||
registerSignalHandlers
|
||||
};
|
||||
1336
package-lock.json
generated
1336
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
13
package.json
13
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "apartment-api",
|
||||
"version": "1.0.1",
|
||||
"version": "1.0.0",
|
||||
"description": "API backend for Country Club Towers & Gardens apartment dashboard",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
@ -8,9 +8,7 @@
|
||||
"dev": "nodemon server.js",
|
||||
"test": "jest",
|
||||
"test:watch": "jest --watch",
|
||||
"test:coverage": "jest --coverage",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix"
|
||||
"test:coverage": "jest --coverage"
|
||||
},
|
||||
"keywords": [
|
||||
"apartments",
|
||||
@ -21,24 +19,17 @@
|
||||
"author": "Stephen",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"axios": "^1.13.5",
|
||||
"cheerio": "^1.2.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"cron-parser": "^5.5.0",
|
||||
"express": "^4.18.2",
|
||||
"express-rate-limit": "^7.1.5",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"mongodb": "^6.3.0",
|
||||
"node-cron": "^4.2.1",
|
||||
"passport": "^0.7.0",
|
||||
"passport-google-oauth20": "^2.0.0",
|
||||
"uuid": "^13.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.18.0",
|
||||
"eslint": "^9.18.0",
|
||||
"globals": "^15.14.0",
|
||||
"jest": "^30.2.0",
|
||||
"mongodb-memory-server": "^11.0.1",
|
||||
"nodemon": "^3.0.2",
|
||||
|
||||
287
routes/admin.js
287
routes/admin.js
@ -71,7 +71,7 @@ router.get('/users', async (req, res) => {
|
||||
order
|
||||
});
|
||||
|
||||
res.json({ data: result });
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('Error fetching users:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch users' });
|
||||
@ -106,11 +106,9 @@ router.get('/users/:id', async (req, res) => {
|
||||
.toArray();
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
user: {
|
||||
...user,
|
||||
recentActivity
|
||||
}
|
||||
user: {
|
||||
...user,
|
||||
recentActivity
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
@ -164,10 +162,8 @@ router.patch('/users/:id', async (req, res) => {
|
||||
});
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
user: updatedUser,
|
||||
message: isActive ? 'User enabled successfully' : 'User disabled successfully'
|
||||
}
|
||||
user: updatedUser,
|
||||
message: isActive ? 'User enabled successfully' : 'User disabled successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error updating user status:', error);
|
||||
@ -251,10 +247,8 @@ router.patch('/users/:id/role', async (req, res) => {
|
||||
: 'User demoted to user successfully';
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
user: updatedUser,
|
||||
message
|
||||
}
|
||||
user: updatedUser,
|
||||
message
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error updating user role:', error);
|
||||
@ -303,14 +297,12 @@ router.get('/users/:id/activity', async (req, res) => {
|
||||
]);
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
activities,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
total,
|
||||
pages: Math.ceil(total / limit) || 0
|
||||
}
|
||||
activities,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
total,
|
||||
pages: Math.ceil(total / limit) || 0
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
@ -440,14 +432,12 @@ router.get('/activity', async (req, res) => {
|
||||
const activities = await collection.aggregate(pipeline).toArray();
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
activities,
|
||||
pagination: {
|
||||
page: validatedPage,
|
||||
limit: validatedLimit,
|
||||
total,
|
||||
pages: Math.ceil(total / validatedLimit) || 0
|
||||
}
|
||||
activities,
|
||||
pagination: {
|
||||
page: validatedPage,
|
||||
limit: validatedLimit,
|
||||
total,
|
||||
pages: Math.ceil(total / validatedLimit) || 0
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
@ -1131,10 +1121,8 @@ router.patch('/settings', async (req, res) => {
|
||||
const updatedSettings = await getSettings(db);
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
settings: updatedSettings,
|
||||
message: 'Settings updated successfully'
|
||||
}
|
||||
settings: updatedSettings,
|
||||
message: 'Settings updated successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error updating settings:', error);
|
||||
@ -1142,236 +1130,5 @@ router.patch('/settings', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// Scraper Endpoints
|
||||
// ============================================================
|
||||
|
||||
const crypto = require('crypto');
|
||||
const {
|
||||
isScraperRunning,
|
||||
acquireLock,
|
||||
releaseLock,
|
||||
getCurrentJobId,
|
||||
getScheduleExpression,
|
||||
getNextScheduledRun
|
||||
} = require('../jobs/scraperJob');
|
||||
const { runScrape } = require('../services/scraperService');
|
||||
const { createLogger } = require('../services/scraperLogger');
|
||||
const scraperConfig = require('../config/scraper');
|
||||
|
||||
// Logger for scraper admin routes
|
||||
const scraperRouteLogger = createLogger('scraper-admin');
|
||||
|
||||
// Rate limiting for manual scrape trigger (per-user, in-memory)
|
||||
const RATE_LIMIT_MAX_REQUESTS = 5;
|
||||
const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
|
||||
const rateLimitStore = new Map();
|
||||
|
||||
/**
|
||||
* Check rate limit for a given user ID.
|
||||
* Returns an object indicating whether the request is allowed.
|
||||
*
|
||||
* @param {string} userId - The user ID to check
|
||||
* @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
|
||||
*/
|
||||
function checkRateLimit(userId) {
|
||||
const now = Date.now();
|
||||
const userKey = userId.toString();
|
||||
|
||||
if (!rateLimitStore.has(userKey)) {
|
||||
rateLimitStore.set(userKey, []);
|
||||
}
|
||||
|
||||
const timestamps = rateLimitStore.get(userKey);
|
||||
|
||||
// Remove timestamps outside the current window
|
||||
const windowStart = now - RATE_LIMIT_WINDOW_MS;
|
||||
const validTimestamps = timestamps.filter(ts => ts > windowStart);
|
||||
rateLimitStore.set(userKey, validTimestamps);
|
||||
|
||||
if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
|
||||
// Calculate when the oldest request in the window will expire
|
||||
const oldestTimestamp = validTimestamps[0];
|
||||
const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
|
||||
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
|
||||
return { allowed: false, retryAfterSeconds };
|
||||
}
|
||||
|
||||
// Record this request
|
||||
validTimestamps.push(now);
|
||||
return { allowed: true, retryAfterSeconds: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the rate limiter (for testing)
|
||||
*/
|
||||
function resetRateLimiter() {
|
||||
rateLimitStore.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/admin/scraper/run
|
||||
* Trigger a manual scrape
|
||||
*
|
||||
* Request body (optional):
|
||||
* - dryRun: boolean - Skip database writes for safe testing
|
||||
* - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
|
||||
*/
|
||||
router.post('/scraper/run', async (req, res) => {
|
||||
try {
|
||||
const db = req.app.locals.db;
|
||||
const { dryRun = false, htmlContent = null } = req.body || {};
|
||||
|
||||
// Check rate limit (per-user, before mutex check)
|
||||
const rateLimitResult = checkRateLimit(req.user._id);
|
||||
if (!rateLimitResult.allowed) {
|
||||
res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
|
||||
return res.status(429).json({
|
||||
error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
|
||||
});
|
||||
}
|
||||
|
||||
// Check if scraper is already running
|
||||
if (isScraperRunning()) {
|
||||
return res.status(409).json({ error: 'Scrape already in progress' });
|
||||
}
|
||||
|
||||
// Generate job ID and acquire lock
|
||||
const jobId = crypto.randomUUID();
|
||||
acquireLock(jobId);
|
||||
|
||||
// Log admin action
|
||||
await logActivity(db, {
|
||||
userId: req.user._id.toString(),
|
||||
action: 'ADMIN_TRIGGER_SCRAPE',
|
||||
metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
|
||||
});
|
||||
|
||||
// Start scrape asynchronously (do not await - return 202 immediately)
|
||||
runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
|
||||
.catch((error) => {
|
||||
scraperRouteLogger.error('Async scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
})
|
||||
.finally(() => releaseLock());
|
||||
|
||||
// Return immediately with job ID
|
||||
res.status(202).json({
|
||||
data: {
|
||||
jobId,
|
||||
status: 'started',
|
||||
dryRun,
|
||||
message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
scraperRouteLogger.error('Error triggering scrape', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to start scrape job' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/admin/scraper/status
|
||||
* Get current scraper status including running state, last run details,
|
||||
* next scheduled run, and schedule expression.
|
||||
*/
|
||||
router.get('/scraper/status', async (req, res) => {
|
||||
try {
|
||||
const db = req.app.locals.db;
|
||||
|
||||
// Get last run from history
|
||||
const lastRunDoc = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
|
||||
.findOne({}, { sort: { startedAt: -1 } });
|
||||
|
||||
const lastRun = lastRunDoc ? {
|
||||
jobId: lastRunDoc.jobId,
|
||||
timestamp: lastRunDoc.startedAt,
|
||||
status: lastRunDoc.status,
|
||||
duration: lastRunDoc.duration,
|
||||
trigger: lastRunDoc.trigger,
|
||||
unitsProcessed: lastRunDoc.unitsProcessed,
|
||||
pricesInserted: lastRunDoc.pricesInserted,
|
||||
errors: lastRunDoc.errors?.length > 0 ? lastRunDoc.errors : null
|
||||
} : null;
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
currentStatus: isScraperRunning() ? 'running' : 'idle',
|
||||
runningJobId: isScraperRunning() ? getCurrentJobId() : null,
|
||||
lastRun,
|
||||
nextScheduledRun: getNextScheduledRun(),
|
||||
schedule: getScheduleExpression()
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
scraperRouteLogger.error('Error fetching scraper status', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
res.status(503).json({ error: 'Service temporarily unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/admin/scraper/history
|
||||
* Get scraper run history with pagination
|
||||
*
|
||||
* Query params:
|
||||
* - limit: Number of records (1-100, default 30)
|
||||
* - offset: Number of records to skip (default 0)
|
||||
*/
|
||||
router.get('/scraper/history', async (req, res) => {
|
||||
try {
|
||||
const db = req.app.locals.db;
|
||||
|
||||
// Parse and validate pagination parameters
|
||||
let limit = parseInt(req.query.limit);
|
||||
let offset = parseInt(req.query.offset);
|
||||
|
||||
// Validate limit
|
||||
if (req.query.limit !== undefined) {
|
||||
if (isNaN(limit) || limit < 1) {
|
||||
return res.status(400).json({ error: 'Limit must be between 1 and 100' });
|
||||
}
|
||||
limit = Math.min(limit, 100);
|
||||
} else {
|
||||
limit = 30;
|
||||
}
|
||||
|
||||
// Validate offset
|
||||
if (req.query.offset !== undefined) {
|
||||
if (isNaN(offset) || offset < 0) {
|
||||
return res.status(400).json({ error: 'Invalid offset parameter' });
|
||||
}
|
||||
} else {
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
|
||||
.find({})
|
||||
.sort({ startedAt: -1 })
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
|
||||
res.json({ data: history });
|
||||
|
||||
} catch (error) {
|
||||
scraperRouteLogger.error('Error fetching scraper history', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
res.status(503).json({ error: 'Service temporarily unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
module.exports.clearStatsCache = clearStatsCache;
|
||||
module.exports.resetRateLimiter = resetRateLimiter;
|
||||
|
||||
66
server.js
66
server.js
@ -11,11 +11,6 @@ const activityRoutes = require('./routes/activity');
|
||||
const adminRoutes = require('./routes/admin');
|
||||
const { createIndexes } = require('./models/user');
|
||||
const { createActivityIndexes } = require('./services/activityLogger');
|
||||
const { createScraperIndexes } = require('./services/scraperService');
|
||||
const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
|
||||
const scraperConfig = require('./config/scraper');
|
||||
|
||||
const { version: API_VERSION } = require('./package.json');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3000;
|
||||
@ -85,32 +80,14 @@ async function connectToMongoDB() {
|
||||
await createIndexes(db);
|
||||
await createActivityIndexes(db);
|
||||
console.log('✅ Passport configured and indexes created');
|
||||
|
||||
// Initialize scraper (indexes + scheduler) - errors logged but don't crash server
|
||||
try {
|
||||
const { createLogger } = require('./services/scraperLogger');
|
||||
const scraperLogger = createLogger('server-init');
|
||||
await createScraperIndexes(db, scraperLogger);
|
||||
if (scraperConfig.SCRAPER_ENABLED) {
|
||||
initializeScheduler(db);
|
||||
console.log('✅ Scraper scheduler initialized');
|
||||
} else {
|
||||
console.log('ℹ️ Scraper scheduling disabled');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('⚠️ Scraper initialization failed (server continues):', error.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to MongoDB:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Register scraper signal handlers for graceful shutdown
|
||||
registerSignalHandlers();
|
||||
|
||||
// Helper function to get today's date in configured timezone
|
||||
const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
|
||||
// Helper function to get today's date in UTC
|
||||
const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
|
||||
|
||||
// Cache for the most recent date with data
|
||||
let cachedLatestDate = null;
|
||||
@ -145,7 +122,7 @@ const getLatestDateWithData = async () => {
|
||||
}
|
||||
|
||||
// Fallback to UTC today if no data found
|
||||
return getTodayDate();
|
||||
return getTodayDateUTC();
|
||||
};
|
||||
|
||||
// Helper function to get yesterday relative to the latest date with data
|
||||
@ -170,14 +147,13 @@ app.use('/auth', authRoutes);
|
||||
// Mount activity routes
|
||||
app.use('/activity', activityRoutes);
|
||||
|
||||
// Mount admin routes (Traefik strips /api prefix, so /api/admin becomes /admin)
|
||||
app.use('/admin', adminRoutes);
|
||||
// Mount admin routes
|
||||
app.use('/api/admin', adminRoutes);
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({
|
||||
status: 'ok',
|
||||
version: API_VERSION,
|
||||
timestamp: new Date().toISOString(),
|
||||
service: 'apartment-api'
|
||||
});
|
||||
@ -315,7 +291,6 @@ app.get('/price-history', requireAuth, async (req, res) => {
|
||||
app.get('/available-units', requireAuth, async (req, res) => {
|
||||
try {
|
||||
const today = await getLatestDateWithData();
|
||||
const yesterday = getYesterdayDate(today);
|
||||
|
||||
// Get units that have prices today (excluding furnished units)
|
||||
const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
|
||||
@ -340,17 +315,11 @@ app.get('/available-units', requireAuth, async (req, res) => {
|
||||
},
|
||||
{
|
||||
$addFields: {
|
||||
currentPriceDoc: {
|
||||
currentPrice: {
|
||||
$arrayElemAt: [
|
||||
{ $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", today] } } },
|
||||
0
|
||||
]
|
||||
},
|
||||
yesterdayPriceDoc: {
|
||||
$arrayElemAt: [
|
||||
{ $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", yesterday] } } },
|
||||
0
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -439,17 +408,6 @@ app.get('/available-units', requireAuth, async (req, res) => {
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
$addFields: {
|
||||
priceChangeToday: {
|
||||
$cond: {
|
||||
if: { $and: [{ $ne: ["$currentPriceDoc.price", null] }, { $ne: ["$yesterdayPriceDoc.price", null] }] },
|
||||
then: { $subtract: ["$currentPriceDoc.price", "$yesterdayPriceDoc.price"] },
|
||||
else: null
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
unitCode: "$unit_code",
|
||||
@ -460,9 +418,7 @@ app.get('/available-units', requireAuth, async (req, res) => {
|
||||
community: "$community",
|
||||
available: "$available",
|
||||
soonest: "$soonest",
|
||||
currentPrice: "$currentPriceDoc.price",
|
||||
yesterdayPrice: "$yesterdayPriceDoc.price",
|
||||
priceChangeToday: { $ifNull: ["$priceChangeToday", null] },
|
||||
currentPrice: "$currentPrice.price",
|
||||
minPrice: "$priceStats.minPrice",
|
||||
maxPrice: "$priceStats.maxPrice",
|
||||
daysAvailable: {
|
||||
@ -644,10 +600,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
|
||||
]).toArray();
|
||||
|
||||
// 4. Historical Comparison - current vs 30/90/365 day averages
|
||||
const tz = scraperConfig.SCRAPER_TIMEZONE;
|
||||
const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
|
||||
const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
|
||||
const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
|
||||
const now = new Date();
|
||||
const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
|
||||
const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
|
||||
const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
|
||||
|
||||
const [currentAvg, avg30, avg90, avg365] = await Promise.all([
|
||||
db.collection(PRICES_COLLECTION).aggregate([
|
||||
|
||||
@ -1,164 +0,0 @@
|
||||
/**
|
||||
* Scraper-specific structured JSON logger
|
||||
* Produces one JSON object per line to stdout
|
||||
*/
|
||||
|
||||
const LEVELS = {
|
||||
info: 'info',
|
||||
warn: 'warn',
|
||||
error: 'error'
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a logger instance scoped to a job ID
|
||||
* @param {string} jobId - Job identifier for correlation
|
||||
* @returns {Object} Logger object with info, warn, error methods
|
||||
*/
|
||||
function createLogger(jobId) {
|
||||
/**
|
||||
* Internal log function
|
||||
* @param {string} level - Log level
|
||||
* @param {string} message - Log message
|
||||
* @param {Object} context - Additional context data
|
||||
*/
|
||||
const log = (level, message, context = {}) => {
|
||||
const entry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
level,
|
||||
message: truncateMessage(message),
|
||||
jobId,
|
||||
context: sanitizeContext(context)
|
||||
};
|
||||
|
||||
// Output as single-line JSON
|
||||
console.log(JSON.stringify(entry));
|
||||
};
|
||||
|
||||
return {
|
||||
info: (message, context) => log(LEVELS.info, message, context),
|
||||
warn: (message, context) => log(LEVELS.warn, message, context),
|
||||
error: (message, context) => log(LEVELS.error, message, context)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate message to prevent log bloat
|
||||
* @param {string} message - Message to truncate
|
||||
* @param {number} maxLength - Maximum length (default 1000)
|
||||
* @returns {string} Truncated message
|
||||
*/
|
||||
function truncateMessage(message, maxLength = 1000) {
|
||||
if (message === null || message === undefined) {
|
||||
return '';
|
||||
}
|
||||
const str = String(message);
|
||||
if (str.length <= maxLength) {
|
||||
return str;
|
||||
}
|
||||
return str.substring(0, maxLength) + '... [truncated]';
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively process an object to handle Buffers, circular references, and sensitive data
|
||||
* @param {*} obj - Object to process
|
||||
* @param {WeakSet} seen - Set of seen objects for circular reference detection
|
||||
* @returns {*} Processed value
|
||||
*/
|
||||
function processValue(obj, seen = new WeakSet()) {
|
||||
// Handle null/undefined
|
||||
if (obj === null) {
|
||||
return null;
|
||||
}
|
||||
if (obj === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle Buffer BEFORE checking for object (Buffer is an object)
|
||||
if (Buffer.isBuffer(obj)) {
|
||||
return `[Buffer: ${obj.length} bytes]`;
|
||||
}
|
||||
|
||||
// Handle strings - check for MongoDB connection strings
|
||||
if (typeof obj === 'string') {
|
||||
if (/mongodb(\+srv)?:\/\//.test(obj)) {
|
||||
return redactConnectionString(obj);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// Handle primitives
|
||||
if (typeof obj !== 'object') {
|
||||
return obj;
|
||||
}
|
||||
|
||||
// Handle circular references
|
||||
if (seen.has(obj)) {
|
||||
return '[Circular]';
|
||||
}
|
||||
seen.add(obj);
|
||||
|
||||
// Handle arrays
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(item => processValue(item, seen));
|
||||
}
|
||||
|
||||
// Handle plain objects
|
||||
const result = {};
|
||||
const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
|
||||
|
||||
for (const key of Object.keys(obj)) {
|
||||
// Check for sensitive keys
|
||||
if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
|
||||
result[key] = '[REDACTED]';
|
||||
} else {
|
||||
result[key] = processValue(obj[key], seen);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize context object for safe logging
|
||||
* - Remove circular references
|
||||
* - Redact sensitive data
|
||||
* - Handle special types (Buffer, undefined)
|
||||
* @param {Object} context - Context object
|
||||
* @returns {Object} Sanitized context
|
||||
*/
|
||||
function sanitizeContext(context) {
|
||||
if (!context || typeof context !== 'object') {
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
return processValue(context);
|
||||
} catch (error) {
|
||||
// If sanitization fails, return empty context
|
||||
return { sanitizationError: 'Failed to sanitize context' };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact credentials from MongoDB connection string
|
||||
* @param {string} uri - Connection string
|
||||
* @returns {string} Redacted string
|
||||
*/
|
||||
function redactConnectionString(uri) {
|
||||
try {
|
||||
// Match mongodb://user:pass@host or mongodb+srv://user:pass@host
|
||||
return uri.replace(
|
||||
/mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
|
||||
'mongodb$1://[user]:[REDACTED]@'
|
||||
);
|
||||
} catch {
|
||||
return '[REDACTED CONNECTION STRING]';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createLogger,
|
||||
LEVELS,
|
||||
// Export internal functions for testing
|
||||
redactConnectionString
|
||||
};
|
||||
@ -1,941 +0,0 @@
|
||||
/**
|
||||
* Scraper Service
|
||||
*
|
||||
* Core scraper logic including HTTP fetching, HTML parsing,
|
||||
* data transformation, and database operations.
|
||||
*/
|
||||
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const crypto = require('crypto');
|
||||
const config = require('../config/scraper');
|
||||
const { createLogger } = require('./scraperLogger');
|
||||
|
||||
/**
|
||||
* Sleep utility for retry delays
|
||||
* @param {number} ms - Milliseconds to sleep
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if an error is retryable
|
||||
* @param {Error} error - Axios error
|
||||
* @returns {boolean} True if should retry
|
||||
*/
|
||||
function isRetryableError(error) {
|
||||
// Network errors (timeout, DNS, connection) don't have a response property
|
||||
if (!error.response) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const status = error.response.status;
|
||||
|
||||
// 5xx server errors are retryable
|
||||
if (status >= 500) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 429 Too Many Requests - do not retry immediately
|
||||
if (status === 429) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4xx client errors - do not retry
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch page HTML with retry logic
|
||||
* @param {string} url - Target URL
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<string>} HTML content
|
||||
* @throws {Error} After all retries exhausted
|
||||
*/
|
||||
async function fetchPage(url, logger) {
|
||||
const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
|
||||
let lastError;
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
||||
try {
|
||||
logger.info('Fetching page', { url, attempt });
|
||||
|
||||
const response = await axios.get(url, {
|
||||
timeout,
|
||||
headers: {
|
||||
'User-Agent': config.USER_AGENT
|
||||
},
|
||||
maxRedirects: 5,
|
||||
validateStatus: (status) => status < 400 // Accept 2xx and 3xx
|
||||
});
|
||||
|
||||
logger.info('Page fetched successfully', {
|
||||
status: response.status,
|
||||
contentLength: response.data.length
|
||||
});
|
||||
|
||||
return response.data;
|
||||
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
|
||||
// Determine if error is retryable
|
||||
const isRetryable = isRetryableError(error);
|
||||
|
||||
logger.warn('Fetch attempt failed', {
|
||||
attempt,
|
||||
errorType: error.name,
|
||||
errorMessage: error.message,
|
||||
statusCode: error.response?.status,
|
||||
isRetryable
|
||||
});
|
||||
|
||||
// Don't retry non-retryable errors (4xx)
|
||||
if (!isRetryable) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Don't wait after last attempt
|
||||
if (attempt <= maxRetries) {
|
||||
const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
|
||||
logger.info('Waiting before retry', { delay });
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse unit data from HTML using cheerio
|
||||
* @param {string} html - HTML content
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
|
||||
*/
|
||||
function parseUnits(html, logger) {
|
||||
const $ = cheerio.load(html);
|
||||
const units = [];
|
||||
|
||||
// Find the main container
|
||||
const container = $('section.spaces__tab-unit');
|
||||
|
||||
if (container.length === 0) {
|
||||
logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
|
||||
return [];
|
||||
}
|
||||
|
||||
// Extract each article element
|
||||
container.find('article').each((index, article) => {
|
||||
const $article = $(article);
|
||||
|
||||
const unit = {
|
||||
// Core identifiers
|
||||
id: $article.attr('data-spaces-id'),
|
||||
unit_code: $article.attr('data-spaces-unit'),
|
||||
unit_id: $article.attr('data-spaces-unit-id'),
|
||||
|
||||
// Physical attributes
|
||||
floor: $article.attr('data-spaces-unit-floor'),
|
||||
area: $article.attr('data-spaces-sort-area'),
|
||||
bed_count: $article.attr('data-spaces-sort-bed'),
|
||||
bath_count: $article.attr('data-spaces-sort-bath'),
|
||||
|
||||
// Pricing
|
||||
price: $article.attr('data-spaces-sort-price'),
|
||||
|
||||
// Availability
|
||||
available: $article.attr('data-spaces-available'),
|
||||
unavailable: $article.attr('data-spaces-unavailable'),
|
||||
soonest: $article.attr('data-spaces-soonest'),
|
||||
date_available: $article.attr('data-spaces-sort-date'),
|
||||
|
||||
// Plan information
|
||||
plan_id: $article.attr('data-spaces-plan-id'),
|
||||
plan_name: $article.attr('data-spaces-sort-plan-name'),
|
||||
|
||||
// Property information
|
||||
obj_type: $article.attr('data-spaces-obj'),
|
||||
community: $article.attr('data-spaces-community'),
|
||||
asset: $article.attr('data-spaces-asset'),
|
||||
|
||||
// URLs
|
||||
href: $article.attr('data-spaces-href'),
|
||||
inventory_href: $article.attr('data-spaces-inventory-href'),
|
||||
|
||||
// Specials
|
||||
specials_content: $article.attr('data-spaces-specials-content')
|
||||
};
|
||||
|
||||
// Extract image URL from nested element if present
|
||||
const imgElement = $article.find('img').first();
|
||||
if (imgElement.length > 0) {
|
||||
unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
|
||||
}
|
||||
|
||||
units.push(unit);
|
||||
});
|
||||
|
||||
logger.info('Units parsed', { count: units.length });
|
||||
|
||||
// Deduplicate by unit_code (in case of duplicate articles)
|
||||
const seen = new Set();
|
||||
const deduplicated = units.filter(unit => {
|
||||
if (!unit.unit_code || seen.has(unit.unit_code)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(unit.unit_code);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (deduplicated.length < units.length) {
|
||||
logger.warn('Duplicate units removed', {
|
||||
original: units.length,
|
||||
deduplicated: deduplicated.length
|
||||
});
|
||||
}
|
||||
|
||||
return deduplicated;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Data Type Conversion Helpers
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Parse string to integer, return null for invalid
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {number|null} Parsed integer or null
|
||||
*/
|
||||
function parseInteger(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
return null;
|
||||
}
|
||||
const parsed = parseInt(value, 10);
|
||||
return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse string to positive integer, return null for 0 or invalid.
|
||||
* Used for fields like area where 0 means "not available".
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {number|null} Parsed positive integer or null
|
||||
*/
|
||||
function parsePositiveInteger(value) {
|
||||
const parsed = parseInteger(value);
|
||||
return parsed === 0 ? null : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse value that could be integer or string identifier.
|
||||
* Returns integer if cleanly parseable, otherwise trimmed string.
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {number|string|null} Parsed integer, trimmed string, or null
|
||||
*/
|
||||
function parseIntegerOrString(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
return null;
|
||||
}
|
||||
const parsed = parseInt(value, 10);
|
||||
// If it parses cleanly to an integer, return integer
|
||||
if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
|
||||
return parsed;
|
||||
}
|
||||
// Otherwise return as trimmed string
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse price, stripping non-numeric characters.
|
||||
* Handles "$1,234" format and "Call for pricing" text.
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {number|null} Parsed price or null
|
||||
*/
|
||||
function parsePrice(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check for "Call for pricing" or similar text
|
||||
if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Strip non-numeric characters except decimal point
|
||||
const cleaned = String(value).replace(/[^0-9.]/g, '');
|
||||
const parsed = parseInt(cleaned, 10);
|
||||
|
||||
if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Negative prices are invalid
|
||||
if (parsed < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse boolean from string.
|
||||
* Handles "true"/"false"/"1"/"0" and actual boolean values.
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {boolean|null} Parsed boolean or null
|
||||
*/
|
||||
function parseBoolean(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
return null;
|
||||
}
|
||||
if (typeof value === 'boolean') {
|
||||
return value;
|
||||
}
|
||||
const str = String(value).toLowerCase().trim();
|
||||
if (str === 'true' || str === '1') return true;
|
||||
if (str === 'false' || str === '0') return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse float value.
|
||||
* Named parseFloatValue to avoid shadowing the global parseFloat.
|
||||
* @param {*} value - Value to parse
|
||||
* @returns {number|null} Parsed float or null
|
||||
*/
|
||||
function parseFloatValue(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim string, return null for empty.
|
||||
* Converts non-string values to string before trimming.
|
||||
* @param {*} value - Value to trim
|
||||
* @returns {string|null} Trimmed string or null
|
||||
*/
|
||||
function trimString(value) {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
const trimmed = String(value).trim();
|
||||
return trimmed === '' ? null : trimmed;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Main Data Type Conversion Function
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Convert unit data types from strings to proper types.
|
||||
* Applies the appropriate parser to each field based on its expected type.
|
||||
* @param {Object} unit - Raw unit object with string values
|
||||
* @returns {Object} Unit object with converted types
|
||||
*/
|
||||
function convertDataTypes(unit) {
|
||||
return {
|
||||
// Integer fields
|
||||
id: parseInteger(unit.id),
|
||||
unit_id: parseIntegerOrString(unit.unit_id),
|
||||
floor: parseInteger(unit.floor),
|
||||
area: parsePositiveInteger(unit.area),
|
||||
bed_count: parseInteger(unit.bed_count),
|
||||
plan_id: parseInteger(unit.plan_id),
|
||||
asset: parseInteger(unit.asset),
|
||||
date_available: parseInteger(unit.date_available),
|
||||
|
||||
// Float fields
|
||||
bath_count: parseFloatValue(unit.bath_count),
|
||||
|
||||
// Price field (special handling)
|
||||
price: parsePrice(unit.price),
|
||||
|
||||
// Boolean fields
|
||||
available: parseBoolean(unit.available),
|
||||
unavailable: parseBoolean(unit.unavailable),
|
||||
|
||||
// String fields (trim and preserve)
|
||||
unit_code: trimString(unit.unit_code),
|
||||
plan_name: trimString(unit.plan_name),
|
||||
soonest: trimString(unit.soonest),
|
||||
obj_type: trimString(unit.obj_type),
|
||||
community: trimString(unit.community),
|
||||
href: trimString(unit.href),
|
||||
inventory_href: trimString(unit.inventory_href),
|
||||
image_url: trimString(unit.image_url),
|
||||
specials_content: trimString(unit.specials_content)
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Date Helpers
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Calculate yesterday's date from a given date string.
|
||||
* @param {string} dateStr - Date in YYYY-MM-DD format
|
||||
* @returns {string} Yesterday's date in YYYY-MM-DD format
|
||||
*/
|
||||
function getYesterday(dateStr) {
|
||||
const date = new Date(dateStr + 'T00:00:00Z');
|
||||
date.setUTCDate(date.getUTCDate() - 1);
|
||||
return date.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Database Operations
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Upsert unit records to database using bulkWrite.
|
||||
* Each unit is matched by unit_code as the unique key.
|
||||
* Sets last_scraped and data_source on every update.
|
||||
* Sets first_seen only on initial insert via $setOnInsert.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Array<Object>} units - Array of unit objects
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object>} Bulk write result
|
||||
*/
|
||||
async function upsertUnits(db, units, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.UNITS);
|
||||
const now = getNow();
|
||||
|
||||
const operations = units.map(unit => ({
|
||||
updateOne: {
|
||||
filter: { unit_code: unit.unit_code },
|
||||
update: {
|
||||
$set: {
|
||||
...unit,
|
||||
last_scraped: now,
|
||||
data_source: 'web_scraper'
|
||||
},
|
||||
$setOnInsert: {
|
||||
first_seen: now
|
||||
}
|
||||
},
|
||||
upsert: true
|
||||
}
|
||||
}));
|
||||
|
||||
if (operations.length === 0) {
|
||||
logger.warn('No units to upsert');
|
||||
return { modifiedCount: 0, upsertedCount: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await collection.bulkWrite(operations, { ordered: false });
|
||||
|
||||
logger.info('Units upserted', {
|
||||
matched: result.matchedCount,
|
||||
modified: result.modifiedCount,
|
||||
upserted: result.upsertedCount
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to upsert units', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert price records for today using bulkWrite with upsert.
|
||||
* Uses unit_code + date_checked as the composite key to prevent
|
||||
* duplicate price records for the same unit on the same day.
|
||||
* Re-running on the same day updates existing records (idempotent).
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Array<Object>} units - Array of unit objects
|
||||
* @param {string} date - Date in YYYY-MM-DD format
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object>} Insert result with insertedCount
|
||||
*/
|
||||
async function insertPrices(db, units, date, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.PRICES);
|
||||
const now = getNow();
|
||||
|
||||
// Filter out units without prices
|
||||
const unitsWithPrices = units.filter(u => u.price !== null);
|
||||
|
||||
const priceRecords = unitsWithPrices.map(unit => ({
|
||||
unit_code: unit.unit_code,
|
||||
date_checked: date,
|
||||
price: unit.price,
|
||||
last_updated: now,
|
||||
data_source: 'web_scraper'
|
||||
}));
|
||||
|
||||
if (priceRecords.length === 0) {
|
||||
logger.warn('No price records to insert');
|
||||
return { insertedCount: 0 };
|
||||
}
|
||||
|
||||
// Use updateOne with upsert to handle re-runs on same day
|
||||
const operations = priceRecords.map(record => ({
|
||||
updateOne: {
|
||||
filter: {
|
||||
unit_code: record.unit_code,
|
||||
date_checked: record.date_checked
|
||||
},
|
||||
update: { $set: record },
|
||||
upsert: true
|
||||
}
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await collection.bulkWrite(operations, { ordered: false });
|
||||
|
||||
logger.info('Prices inserted', {
|
||||
inserted: result.upsertedCount,
|
||||
updated: result.modifiedCount
|
||||
});
|
||||
|
||||
return { insertedCount: result.upsertedCount + result.modifiedCount };
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to insert prices', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark units not in current scrape as stale/unavailable.
|
||||
* Uses updateMany to set available: false and marked_stale_date
|
||||
* for all units whose unit_code is NOT in the current scrape
|
||||
* and that are currently available.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Set<string>} currentUnitCodes - Unit codes from current scrape
|
||||
* @param {string} date - Current date in YYYY-MM-DD format
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object>} Update result
|
||||
*/
|
||||
async function markStaleUnits(db, currentUnitCodes, date, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.UNITS);
|
||||
|
||||
try {
|
||||
const result = await collection.updateMany(
|
||||
{
|
||||
unit_code: { $nin: [...currentUnitCodes] },
|
||||
available: true
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
available: false,
|
||||
marked_stale_date: date
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
logger.info('Stale units marked', { count: result.modifiedCount });
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to mark stale units', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update daily summary document.
|
||||
* Upserts by date field (YYYY-MM-DD format).
|
||||
* Fetches yesterday's summary for comparison metrics.
|
||||
* Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} summaryData - Summary data
|
||||
* @param {string} summaryData.date - Date in YYYY-MM-DD format
|
||||
* @param {Array<string>} summaryData.newUnits - Unit codes added today
|
||||
* @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
|
||||
* @param {number} summaryData.staleUnitsCount - Count of stale units
|
||||
* @param {number} summaryData.totalAvailable - Total available units today
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object>} MongoDB updateOne result
|
||||
*/
|
||||
async function updateDailySummary(db, summaryData, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
|
||||
const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
|
||||
|
||||
// Calculate yesterday's date for comparison
|
||||
const yesterday = getYesterday(date);
|
||||
|
||||
// Get yesterday's summary for comparison
|
||||
const yesterdaySummary = await collection.findOne({ date: yesterday });
|
||||
const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
|
||||
|
||||
const summary = {
|
||||
date,
|
||||
timestamp: getNow(),
|
||||
new_units: newUnits,
|
||||
rented_units: rentedUnits,
|
||||
stale_units: [], // Stale units list is not tracked per PRD
|
||||
new_units_count: newUnits.length,
|
||||
rented_units_count: rentedUnits.length,
|
||||
stale_units_count: staleUnitsCount,
|
||||
net_change: newUnits.length - rentedUnits.length,
|
||||
total_available_today: totalAvailable,
|
||||
total_available_yesterday: yesterdayTotal,
|
||||
turnover_rate: yesterdayTotal > 0
|
||||
? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
|
||||
: 0
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await collection.updateOne(
|
||||
{ date },
|
||||
{ $set: summary },
|
||||
{ upsert: true }
|
||||
);
|
||||
|
||||
logger.info('Daily summary updated', {
|
||||
date,
|
||||
newUnits: newUnits.length,
|
||||
rentedUnits: rentedUnits.length,
|
||||
totalAvailable
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to update daily summary', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Additional Helpers for runScrape Orchestration
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Get today's date in YYYY-MM-DD format using the configured timezone.
|
||||
* @returns {string} Today's date string
|
||||
*/
|
||||
function getToday() {
|
||||
return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current timestamp in the configured timezone as an ISO-like string.
|
||||
* Returns format: YYYY-MM-DDTHH:mm:ss.mmm (no Z suffix, local time).
|
||||
* @returns {string} Current timestamp in configured timezone
|
||||
*/
|
||||
function getNow() {
|
||||
const now = new Date();
|
||||
const formatter = new Intl.DateTimeFormat('en-CA', {
|
||||
timeZone: config.SCRAPER_TIMEZONE,
|
||||
year: 'numeric', month: '2-digit', day: '2-digit',
|
||||
hour: '2-digit', minute: '2-digit', second: '2-digit',
|
||||
hour12: false
|
||||
});
|
||||
const parts = Object.fromEntries(
|
||||
formatter.formatToParts(now).map(({ type, value }) => [type, value])
|
||||
);
|
||||
const ms = String(now.getMilliseconds()).padStart(3, '0');
|
||||
return `${parts.year}-${parts.month}-${parts.day}T${parts.hour}:${parts.minute}:${parts.second}.${ms}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the set of unit codes that had price records yesterday.
|
||||
* Used to calculate new and rented units by comparison.
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {string} today - Today's date in YYYY-MM-DD format
|
||||
* @returns {Promise<Set<string>>} Set of unit codes from yesterday
|
||||
*/
|
||||
async function getYesterdayUnitCodes(db, today) {
|
||||
const yesterday = getYesterday(today);
|
||||
const collection = db.collection(config.COLLECTIONS.PRICES);
|
||||
|
||||
const yesterdayRecords = await collection
|
||||
.find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
|
||||
.toArray();
|
||||
|
||||
return new Set(yesterdayRecords.map(r => r.unit_code));
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Error Sanitization
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Sanitize an error object to remove sensitive information before storage.
|
||||
* Removes file paths, connection strings, and credential patterns while
|
||||
* preserving the error type and a useful general description for debugging.
|
||||
*
|
||||
* @param {Error} error - Error object to sanitize
|
||||
* @returns {Object} Sanitized error with name, message, and optionally stack
|
||||
*/
|
||||
function sanitizeError(error) {
|
||||
const sanitized = {
|
||||
name: error.name || 'Error',
|
||||
message: sanitizeMessage(error.message || ''),
|
||||
};
|
||||
|
||||
if (error.stack) {
|
||||
sanitized.stack = sanitizeMessage(error.stack);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a string message by removing sensitive patterns.
|
||||
* @param {string} message - Raw error message
|
||||
* @returns {string} Sanitized message
|
||||
*/
|
||||
function sanitizeMessage(message) {
|
||||
let result = message;
|
||||
|
||||
// Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
|
||||
result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
|
||||
|
||||
// Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
|
||||
result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
|
||||
|
||||
// Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
|
||||
result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
|
||||
|
||||
// Remove Windows-style absolute paths (C:\..., D:\...)
|
||||
result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Scraper Run History
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Record scraper run to history collection.
|
||||
* Called in the finally block of runScrape() to persist run metadata.
|
||||
* This function must NOT throw errors - history recording should never break the scraper.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<Object|null>} Insert result, or null on failure
|
||||
*/
|
||||
async function recordScraperRun(db, runData, logger) {
|
||||
try {
|
||||
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
|
||||
const result = await collection.insertOne({
|
||||
...runData,
|
||||
recordedAt: getNow()
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
// Log but don't throw - recording history should not break scraper
|
||||
logger.error('Failed to record scraper run', { errorMessage: error.message });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Index Management
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Create indexes on the scraper_runs collection.
|
||||
* Should be called once during application startup.
|
||||
* Idempotent - safe to call multiple times.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function createScraperIndexes(db, logger) {
|
||||
const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
|
||||
|
||||
// Compound index for querying runs by status sorted by most recent
|
||||
await collection.createIndex(
|
||||
{ status: 1, startedAt: -1 },
|
||||
{ name: 'status_startedAt' }
|
||||
);
|
||||
|
||||
// Index for sorting all runs by start time (most recent first)
|
||||
await collection.createIndex(
|
||||
{ startedAt: -1 },
|
||||
{ name: 'startedAt_desc' }
|
||||
);
|
||||
|
||||
logger.info('Scraper indexes created successfully');
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Main Orchestration Function
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Execute a complete scrape operation.
|
||||
* Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
|
||||
*
|
||||
* @param {Db} db - MongoDB database instance
|
||||
* @param {Object} options - Scrape options
|
||||
* @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
|
||||
* @param {string} [options.jobId] - Optional job ID (generated if not provided)
|
||||
* @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
|
||||
* @param {string} [options.htmlContent] - Use provided HTML instead of fetching
|
||||
* @returns {Promise<Object>} Scrape result with status and metrics
|
||||
*/
|
||||
async function runScrape(db, options = {}) {
|
||||
const jobId = options.jobId || crypto.randomUUID();
|
||||
const trigger = options.trigger || 'manual';
|
||||
const dryRun = options.dryRun || false;
|
||||
const htmlContent = options.htmlContent || null;
|
||||
const logger = createLogger(jobId);
|
||||
const startTime = Date.now();
|
||||
|
||||
let result = {
|
||||
jobId,
|
||||
trigger,
|
||||
dryRun,
|
||||
status: 'running',
|
||||
startedAt: getNow(),
|
||||
completedAt: null,
|
||||
duration: null,
|
||||
unitsProcessed: 0,
|
||||
pricesInserted: 0,
|
||||
newUnitsCount: 0,
|
||||
rentedUnitsCount: 0,
|
||||
staleUnitsCount: 0,
|
||||
errors: []
|
||||
};
|
||||
|
||||
try {
|
||||
logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
|
||||
|
||||
// Step 1: Fetch HTML (or use provided content for testing)
|
||||
const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
|
||||
|
||||
// Step 2: Parse units
|
||||
const rawUnits = parseUnits(html, logger);
|
||||
|
||||
if (rawUnits.length === 0) {
|
||||
logger.warn('No units found in HTML - possible structure change');
|
||||
result.errors.push('No units found in HTML');
|
||||
}
|
||||
|
||||
// Step 3: Convert data types
|
||||
const units = rawUnits.map(unit => convertDataTypes(unit));
|
||||
result.unitsProcessed = units.length;
|
||||
|
||||
// Step 4: Database operations
|
||||
const today = getToday();
|
||||
|
||||
// Get yesterday's unit codes for comparison
|
||||
const yesterdayUnits = await getYesterdayUnitCodes(db, today);
|
||||
|
||||
// Determine new and rented units
|
||||
const currentUnitCodes = new Set(units.map(u => u.unit_code));
|
||||
const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
|
||||
const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
|
||||
|
||||
result.newUnitsCount = newUnits.length;
|
||||
result.rentedUnitsCount = rentedUnits.length;
|
||||
|
||||
// Database operations (skip if dryRun)
|
||||
if (dryRun) {
|
||||
logger.info('Dry run mode - skipping database writes', {
|
||||
wouldUpsert: units.length,
|
||||
wouldInsertPrices: units.filter(u => u.price !== null).length
|
||||
});
|
||||
result.pricesInserted = 0;
|
||||
result.staleUnitsCount = 0;
|
||||
} else {
|
||||
// Upsert units
|
||||
await upsertUnits(db, units, logger);
|
||||
|
||||
// Insert prices
|
||||
const pricesResult = await insertPrices(db, units, today, logger);
|
||||
result.pricesInserted = pricesResult.insertedCount;
|
||||
|
||||
// Mark stale units
|
||||
const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
|
||||
result.staleUnitsCount = staleResult.modifiedCount;
|
||||
|
||||
// Update daily summary
|
||||
await updateDailySummary(db, {
|
||||
date: today,
|
||||
newUnits: newUnits.map(u => u.unit_code),
|
||||
rentedUnits,
|
||||
staleUnitsCount: result.staleUnitsCount,
|
||||
totalAvailable: units.length
|
||||
}, logger);
|
||||
}
|
||||
|
||||
result.status = 'success';
|
||||
|
||||
} catch (error) {
|
||||
const cleanError = sanitizeError(error);
|
||||
logger.error('Scrape failed', {
|
||||
errorType: cleanError.name,
|
||||
errorMessage: cleanError.message
|
||||
});
|
||||
result.status = 'failed';
|
||||
result.errors.push(cleanError.message);
|
||||
|
||||
} finally {
|
||||
result.completedAt = getNow();
|
||||
result.duration = Date.now() - startTime;
|
||||
|
||||
// Record run to history (always runs, even on failure)
|
||||
await recordScraperRun(db, result, logger);
|
||||
|
||||
logger.info('Scrape completed', {
|
||||
status: result.status,
|
||||
duration: result.duration,
|
||||
unitsProcessed: result.unitsProcessed,
|
||||
pricesInserted: result.pricesInserted
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runScrape,
|
||||
fetchPage,
|
||||
parseUnits,
|
||||
convertDataTypes,
|
||||
upsertUnits,
|
||||
insertPrices,
|
||||
markStaleUnits,
|
||||
updateDailySummary,
|
||||
recordScraperRun,
|
||||
createScraperIndexes,
|
||||
// Export helpers for testing
|
||||
getToday,
|
||||
getNow,
|
||||
getYesterdayUnitCodes,
|
||||
getYesterday,
|
||||
parseInteger,
|
||||
parsePositiveInteger,
|
||||
parseIntegerOrString,
|
||||
parsePrice,
|
||||
parseBoolean,
|
||||
parseFloatValue,
|
||||
trimString,
|
||||
isRetryableError,
|
||||
sleep,
|
||||
sanitizeError
|
||||
};
|
||||
Reference in New Issue
Block a user