Implement Phase 2 activity monitoring API

- Add GET /api/admin/activity with pagination and filters
- Add GET /api/admin/users/:id/activity for user-specific history
- Add GET /api/admin/activity/export for CSV export
- Implement $lookup aggregation for user info (name, email)
- Add date range filtering with validation
- Export limited to 10,000 records with truncation headers
- All 50 Phase 2 tests passing

.github/workflows/deploy.yml

@@ -1,29 +1,14 @@
-name: CI/CD Pipeline - Apartment API
+name: Deploy Apartment API
 
 on:
   push:
     branches: [ main ]
-  pull_request:
   workflow_dispatch:
 
-env:
-  NODE_VERSION: '20'
-  IMAGE_NAME: apartment-api
-
 jobs:
-  # ============================================================
+  deploy:
-  # CI Job - Lint + Test in a single job (one checkout, one npm ci)
-  # ============================================================
-  ci:
-    name: Lint & Test
     runs-on: ubuntu-latest
     
-    outputs:
-      lint_status: ${{ steps.lint.outcome }}
-      lint_output: ${{ steps.lint.outputs.lint_output }}
-      test_status: ${{ steps.test.outcome }}
-      test_output: ${{ steps.test.outputs.test_output }}
-
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
@@ -31,343 +16,45 @@ jobs:
     - name: Setup Node.js
       uses: actions/setup-node@v4
       with:
-          node-version: ${{ env.NODE_VERSION }}
+        node-version: '18'
         cache: 'npm'
         
     - name: Install dependencies
       run: npm ci
       
-      - name: Run ESLint
+    - name: Build Docker image
-        id: lint
-        continue-on-error: true
       run: |
-          set +e
+        docker build -t apartment-api:${{ github.sha }} .
-          OUTPUT=$(npm run lint 2>&1)
+        docker tag apartment-api:${{ github.sha }} apartment-api:latest
-          EXIT_CODE=$?
-          # Truncate before writing to GITHUB_OUTPUT to prevent
-          # "argument list too long" in downstream jobs
-          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
-          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-          exit $EXIT_CODE
         
-      - name: Run tests
+    - name: Deploy to server
-        id: test
-        continue-on-error: true
       run: |
-          set +e
+        # Copy files to deployment directory
-          OUTPUT=$(npm test -- --runInBand 2>&1)
+        mkdir -p /media/stephen/Storage_Linux/infrastructure/services/apartment-api
-          EXIT_CODE=$?
+        cp -r . /media/stephen/Storage_Linux/infrastructure/services/apartment-api/
-          # Truncate before writing to GITHUB_OUTPUT to prevent
-          # "argument list too long" in downstream jobs
-          echo "test_output<<EOF" >> $GITHUB_OUTPUT
-          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-          exit $EXIT_CODE
-        env:
-          JWT_SECRET: test-jwt-secret-for-ci
-          NODE_ENV: test
-
-  # ============================================================
-  # Notify Job - Send results to n8n webhook
-  # ============================================================
-  notify:
-    name: Send Webhook Notification
-    runs-on: ubuntu-latest
-    needs: [ci]
-    if: always()
-
-    steps:
-      - name: Send results to n8n webhook
-        env:
-          LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
-          TEST_STATUS: ${{ needs.ci.outputs.test_status }}
-          RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
-          RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
-          GH_REPO: ${{ github.repository }}
-          GH_BRANCH: ${{ github.head_ref || github.ref_name }}
-          GH_SHA: ${{ github.sha }}
-          GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
-          GH_ACTOR: ${{ github.actor }}
-          GH_EVENT: ${{ github.event_name }}
-          GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
-          GH_RUN_ID: ${{ github.run_id }}
-          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
-        run: |
-          # Determine overall status
-          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
-            OVERALL_STATUS="success"
-          else
-            OVERALL_STATUS="failure"
-          fi
-
-          # Outputs are already truncated at the source (ci job)
-          LINT_OUTPUT="$RAW_LINT_OUTPUT"
-          TEST_OUTPUT="$RAW_TEST_OUTPUT"
-
-          # Build JSON payload
-          PAYLOAD=$(jq -n \
-            --arg repo "$GH_REPO" \
-            --arg branch "$GH_BRANCH" \
-            --arg commit "$GH_SHA" \
-            --arg commit_short "${GH_SHA:0:7}" \
-            --arg commit_message "$GH_COMMIT_MSG" \
-            --arg author "$GH_ACTOR" \
-            --arg event "$GH_EVENT" \
-            --arg pr_number "$GH_PR_NUMBER" \
-            --arg run_id "$GH_RUN_ID" \
-            --arg run_url "$GH_RUN_URL" \
-            --arg overall_status "$OVERALL_STATUS" \
-            --arg lint_status "$LINT_STATUS" \
-            --arg lint_output "$LINT_OUTPUT" \
-            --arg test_status "$TEST_STATUS" \
-            --arg test_output "$TEST_OUTPUT" \
-            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            '{
-              repository: $repo,
-              branch: $branch,
-              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
-              commit: {
-                sha: $commit,
-                short_sha: $commit_short,
-                message: $commit_message,
-                author: $author
-              },
-              event: $event,
-              run: {
-                id: $run_id,
-                url: $run_url
-              },
-              status: $overall_status,
-              results: {
-                lint: {
-                  status: $lint_status,
-                  output: (if $lint_status != "success" then $lint_output else null end)
-                },
-                test: {
-                  status: $test_status,
-                  output: (if $test_status != "success" then $test_output else null end)
-                }
-              },
-              timestamp: $timestamp
-            }')
-
-          # Send webhook
-          curl -X POST \
-            -H "Content-Type: application/json" \
-            -d "$PAYLOAD" \
-            "$WEBHOOK_URL" \
-            --fail --silent --show-error
-
-      - name: Fail if lint or tests failed
-        if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
-        run: |
-          echo "❌ Pipeline failed:"
-          echo "   Lint: ${{ needs.ci.outputs.lint_status }}"
-          echo "   Test: ${{ needs.ci.outputs.test_status }}"
-          exit 1
-
-  # ============================================================
-  # Dependency Scan Job - Runs in parallel with tests
-  # ============================================================
-  scan-deps:
-    name: Scan Dependencies
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install Trivy
-        run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
-
-      - name: Scan dependencies for vulnerabilities
-        run: |
-          trivy fs \
-            --exit-code 1 \
-            --ignore-unfixed \
-            --severity CRITICAL,HIGH \
-            --scanners vuln \
-            .
-
-  # ============================================================
-  # Build & Push Job - Build image and push to Harbor
-  # ============================================================
-  build:
-    name: Build & Push Image
-    runs-on: ubuntu-latest
-    needs: [ci, scan-deps, notify]
-    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
-
-    outputs:
-      image_tag: ${{ steps.set-tag.outputs.tag }}
-      full_image: ${{ steps.set-tag.outputs.full_image }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set image tag
-        id: set-tag
-        run: |
-          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
-          echo "tag=${SHORT_SHA}" >> $GITHUB_OUTPUT
-          echo "full_image=${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${SHORT_SHA}" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Log in to Harbor
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.HARBOR_REGISTRY }}
-          username: ${{ secrets.HARBOR_USERNAME }}
-          password: ${{ secrets.HARBOR_PASSWORD }}
-
-      - name: Build and push Docker image
-        id: build-push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
-            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:latest
-
-      # ============================================================
-      # Vulnerability Scanning with Trivy
-      # ============================================================
-      - name: Install Trivy
-        run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
-
-      - name: Run Trivy vulnerability scanner
-        run: |
-          trivy image \
-            --exit-code 1 \
-            --ignore-unfixed \
-            --vuln-type os,library \
-            --severity CRITICAL,HIGH \
-            --format table \
-            ${{ steps.set-tag.outputs.full_image }}
-
-      - name: Run Trivy and output SARIF
-        if: always()
-        run: |
-          trivy image \
-            --format sarif \
-            --output trivy-results.sarif \
-            ${{ steps.set-tag.outputs.full_image }} || true
-
-      # ============================================================
-      # SBOM Generation with Syft
-      # ============================================================
-      - name: Install Syft
-        run: |
-          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
-
-      - name: Generate SBOM with Syft
-        run: |
-          syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json
-
-      # ============================================================
-      # Image Signing with Cosign
-      # ============================================================
-      - name: Install Cosign
-        uses: sigstore/cosign-installer@v3
-
-      - name: Sign image with Cosign
-        env:
-          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
-          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
-        run: |
-          cosign sign --key env://COSIGN_PRIVATE_KEY \
-            --yes \
-            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
-
-      # ============================================================
-      # Attach SBOM to image in Harbor
-      # ============================================================
-      - name: Attach SBOM to image
-        env:
-          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
-          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
-        run: |
-          cosign attest --key env://COSIGN_PRIVATE_KEY \
-            --type spdxjson \
-            --predicate sbom.spdx.json \
-            --yes \
-            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
-
-  # ============================================================
-  # Deploy Job - Pull image and restart on production server
-  # ============================================================
-  deploy:
-    name: Deploy to Production
-    runs-on: ubuntu-latest
-    needs: build
-    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
-
-    steps:
-      - name: Deploy via SSH
-        uses: appleboy/ssh-action@v1.0.3
-        with:
-          host: ${{ secrets.SSH_HOST }}
-          username: ${{ secrets.SSH_USER }}
-          key: ${{ secrets.SSH_PRIVATE_KEY }}
-          port: ${{ secrets.SSH_PORT || 22 }}
-          envs: IMAGE
-          script_stop: true
-          script: |
-            set -e
-
-            # Image to deploy (using specific SHA tag, not latest)
-            IMAGE="${{ needs.build.outputs.full_image }}"
-            echo "Deploying image: $IMAGE"
         
         # Navigate to deployment directory
-            cd ${{ secrets.DEPLOY_PATH }}
+        cd /media/stephen/Storage_Linux/infrastructure/services/apartment-api
         
-            # Pull the specific image (using pre-configured Docker credentials)
+        # Stop existing container if running
-            docker pull "$IMAGE"
+        docker compose down || true
         
-            # Update and restart with new image
+        # Start new container
-            export IMAGE
         docker compose up -d
         
         # Clean up old images
         docker image prune -f
         
-            # Wait for container to be healthy
-            echo "Waiting for container to start..."
-            sleep 10
-
-            # Verify container is running
-            if docker compose ps | grep -q "Up"; then
-              echo "Container is running successfully"
-            else
-              echo "ERROR: Container failed to start"
-              docker compose logs --tail=50
-              exit 1
-            fi
-
     - name: Verify deployment
-        uses: appleboy/ssh-action@v1.0.3
+      run: |
-        with:
+        # Wait for container to start
-          host: ${{ secrets.SSH_HOST }}
+        sleep 15
-          username: ${{ secrets.SSH_USER }}
-          key: ${{ secrets.SSH_PRIVATE_KEY }}
-          port: ${{ secrets.SSH_PORT || 22 }}
-          script: |
-            # Test health endpoint
-            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://apartments.maverickapplications.com/api/health || echo "000")
         
-            if [ "$HTTP_CODE" = "200" ]; then
+        # Check if container is running
-              echo "Health check passed (HTTP $HTTP_CODE)"
+        docker ps | grep apartment-api
-            else
+        
-              echo "WARNING: Health check returned HTTP $HTTP_CODE"
+        # Test health endpoint
-              echo "Container may still be initializing..."
+        curl -f http://localhost:3000/health || echo "API may still be starting..."
-              exit 1
+        
-            fi
+        # Test API endpoint
+        curl -f http://localhost:3000/api/daily-summary || echo "API may still be loading data..."

Dockerfile

@@ -1,12 +1,8 @@
-FROM node:20-alpine
+FROM node:18-alpine
 
 # Set working directory
 WORKDIR /app
 
-# Update Alpine packages and npm to fix security vulnerabilities
-RUN apk upgrade --no-cache && \
-    npm install -g npm@latest
-
 # Install Curl
 RUN apk add --no-cache curl
 
@@ -14,7 +10,7 @@ RUN apk add --no-cache curl
 COPY package*.json ./
 
 # Install dependencies
-RUN npm ci --omit=dev
+RUN npm ci --only=production
 
 # Copy source code
 COPY . .

__tests__/phase1-foundation.test.js

@@ -361,13 +361,13 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.users).toBeDefined();
+        expect(response.body.users).toBeDefined();
-        expect(Array.isArray(response.body.data.users)).toBe(true);
+        expect(Array.isArray(response.body.users)).toBe(true);
-        expect(response.body.data.pagination).toBeDefined();
+        expect(response.body.pagination).toBeDefined();
-        expect(response.body.data.pagination.page).toBe(1);
+        expect(response.body.pagination.page).toBe(1);
-        expect(response.body.data.pagination.limit).toBe(20);
+        expect(response.body.pagination.limit).toBe(20);
-        expect(response.body.data.pagination.total).toBeGreaterThan(0);
+        expect(response.body.pagination.total).toBeGreaterThan(0);
-        expect(response.body.data.pagination.pages).toBeDefined();
+        expect(response.body.pagination.pages).toBeDefined();
       });
 
       it('should support page parameter', async () => {
@@ -383,7 +383,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.pagination.page).toBe(2);
+        expect(response.body.pagination.page).toBe(2);
       });
 
       it('should support limit parameter with max 100', async () => {
@@ -392,7 +392,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.pagination.limit).toBe(50);
+        expect(response.body.pagination.limit).toBe(50);
       });
 
       it('should cap limit at 100', async () => {
@@ -401,7 +401,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.pagination.limit).toBeLessThanOrEqual(100);
+        expect(response.body.pagination.limit).toBeLessThanOrEqual(100);
       });
 
       it('should support search parameter for name', async () => {
@@ -413,8 +413,8 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.users.some(u => u.name.includes('John'))).toBe(true);
+        expect(response.body.users.some(u => u.name.includes('John'))).toBe(true);
-        expect(response.body.data.users.every(u =>
+        expect(response.body.users.every(u =>
           u.name.toLowerCase().includes('john') ||
           u.email.toLowerCase().includes('john')
         )).toBe(true);
@@ -428,7 +428,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.users.some(u => u.email.includes('unique-test'))).toBe(true);
+        expect(response.body.users.some(u => u.email.includes('unique-test'))).toBe(true);
       });
 
       it('should support status filter "active"', async () => {
@@ -440,7 +440,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.users.every(u => u.isActive === true)).toBe(true);
+        expect(response.body.users.every(u => u.isActive === true)).toBe(true);
       });
 
       it('should support status filter "disabled"', async () => {
@@ -451,7 +451,7 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.users.every(u => u.isActive === false)).toBe(true);
+        expect(response.body.users.every(u => u.isActive === false)).toBe(true);
       });
 
       it('should support sort parameter', async () => {
@@ -472,7 +472,7 @@ describe('Phase 1: Foundation', () => {
 
         expect(response.status).toBe(200);
         // Newest first when sorted descending
-        const createdDates = response.body.data.users.map(u => new Date(u.createdAt));
+        const createdDates = response.body.users.map(u => new Date(u.createdAt));
         for (let i = 0; i < createdDates.length - 1; i++) {
           expect(createdDates[i] >= createdDates[i + 1]).toBe(true);
         }
@@ -485,7 +485,7 @@ describe('Phase 1: Foundation', () => {
 
         expect(response.status).toBe(200);
         // Verify ascending order
-        const counts = response.body.data.users.map(u => u.loginCount);
+        const counts = response.body.users.map(u => u.loginCount);
         for (let i = 0; i < counts.length - 1; i++) {
           expect(counts[i] <= counts[i + 1]).toBe(true);
         }
@@ -517,11 +517,11 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user).toBeDefined();
+        expect(response.body.user).toBeDefined();
-        expect(response.body.data.user._id.toString()).toBe(testUser._id.toString());
+        expect(response.body.user._id.toString()).toBe(testUser._id.toString());
-        expect(response.body.data.user.email).toBe(testUser.email);
+        expect(response.body.user.email).toBe(testUser.email);
-        expect(response.body.data.user.name).toBe(testUser.name);
+        expect(response.body.user.name).toBe(testUser.name);
-        expect(response.body.data.user.role).toBe(testUser.role);
+        expect(response.body.user.role).toBe(testUser.role);
       });
 
       it('should include recent activity for the user', async () => {
@@ -550,8 +550,8 @@ describe('Phase 1: Foundation', () => {
           .set('Cookie', [`auth_token=${adminToken}`]);
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user.recentActivity).toBeDefined();
+        expect(response.body.user.recentActivity).toBeDefined();
-        expect(Array.isArray(response.body.data.user.recentActivity)).toBe(true);
+        expect(Array.isArray(response.body.user.recentActivity)).toBe(true);
       });
 
       it('should return 404 for non-existent user ID', async () => {
@@ -601,9 +601,9 @@ describe('Phase 1: Foundation', () => {
           .send({ isActive: false });
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user).toBeDefined();
+        expect(response.body.user).toBeDefined();
-        expect(response.body.data.user.isActive).toBe(false);
+        expect(response.body.user.isActive).toBe(false);
-        expect(response.body.data.message).toContain('disabled');
+        expect(response.body.message).toContain('disabled');
 
         // Verify in database
         const dbUser = await db.collection('users').findOne({ _id: testUser._id });
@@ -620,9 +620,9 @@ describe('Phase 1: Foundation', () => {
           .send({ isActive: true });
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user).toBeDefined();
+        expect(response.body.user).toBeDefined();
-        expect(response.body.data.user.isActive).toBe(true);
+        expect(response.body.user.isActive).toBe(true);
-        expect(response.body.data.message).toContain('enabled');
+        expect(response.body.message).toContain('enabled');
       });
 
       it('should set disabledAt timestamp when disabling', async () => {
@@ -729,9 +729,9 @@ describe('Phase 1: Foundation', () => {
           .send({ role: 'admin' });
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user).toBeDefined();
+        expect(response.body.user).toBeDefined();
-        expect(response.body.data.user.role).toBe('admin');
+        expect(response.body.user.role).toBe('admin');
-        expect(response.body.data.message).toContain('promoted');
+        expect(response.body.message).toContain('promoted');
 
         // Verify in database
         const dbUser = await db.collection('users').findOne({ _id: testUser._id });
@@ -748,9 +748,9 @@ describe('Phase 1: Foundation', () => {
           .send({ role: 'user' });
 
         expect(response.status).toBe(200);
-        expect(response.body.data.user).toBeDefined();
+        expect(response.body.user).toBeDefined();
-        expect(response.body.data.user.role).toBe('user');
+        expect(response.body.user.role).toBe('user');
-        expect(response.body.data.message).toContain('demoted');
+        expect(response.body.message).toContain('demoted');
       });
 
       it('should return 400 when trying to demote yourself from admin', async () => {
@@ -846,7 +846,7 @@ describe('Phase 1: Foundation', () => {
         expect(response.status).toBe(200);
 
         // Check that sensitive fields are not exposed
-        const userInResponse = response.body.data.users.find(
+        const userInResponse = response.body.users.find(
           u => u._id.toString() === testUser._id.toString()
         );
 

__tests__/phase2-activity.test.js

@@ -94,8 +94,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data).toHaveProperty('activities');
+        expect(res.body).toHaveProperty('activities');
-        expect(res.body.data).toHaveProperty('pagination');
+        expect(res.body).toHaveProperty('pagination');
       });
     });
 
@@ -119,8 +119,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(50);
+        expect(res.body.activities).toHaveLength(50);
-        expect(res.body.data.pagination).toEqual(expect.objectContaining({
+        expect(res.body.pagination).toEqual(expect.objectContaining({
           page: 1,
           limit: 50,
           total: 100,
@@ -146,8 +146,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(50);
+        expect(res.body.activities).toHaveLength(50);
-        expect(res.body.data.pagination.page).toBe(2);
+        expect(res.body.pagination.page).toBe(2);
       });
 
       it('should respect custom limit parameter', async () => {
@@ -168,9 +168,9 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(25);
+        expect(res.body.activities).toHaveLength(25);
-        expect(res.body.data.pagination.limit).toBe(25);
+        expect(res.body.pagination.limit).toBe(25);
-        expect(res.body.data.pagination.pages).toBe(4);
+        expect(res.body.pagination.pages).toBe(4);
       });
 
       it('should enforce maximum limit of 200', async () => {
@@ -183,7 +183,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.pagination.limit).toBeLessThanOrEqual(200);
+        expect(res.body.pagination.limit).toBeLessThanOrEqual(200);
       });
 
       it('should return empty array for page beyond available data', async () => {
@@ -204,7 +204,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(0);
+        expect(res.body.activities).toHaveLength(0);
       });
     });
 
@@ -231,8 +231,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(20);
+        expect(res.body.activities).toHaveLength(20);
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity.userId.toString()).toBe(user1._id.toString());
         });
       });
@@ -256,8 +256,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(10);
+        expect(res.body.activities).toHaveLength(10);
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity.action).toBe('login');
         });
       });
@@ -286,7 +286,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(new Date(activity.timestamp).getTime()).toBeGreaterThanOrEqual(startDate.getTime());
         });
       });
@@ -315,7 +315,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(new Date(activity.timestamp).getTime()).toBeLessThanOrEqual(endDate.getTime());
         });
       });
@@ -340,7 +340,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity.userId.toString()).toBe(user1._id.toString());
           expect(activity.action).toBe('page_view');
         });
@@ -356,7 +356,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(0);
+        expect(res.body.activities).toHaveLength(0);
       });
     });
 
@@ -377,7 +377,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity).toHaveProperty('userName');
           expect(activity.userName).toBe('Test User Name');
         });
@@ -399,7 +399,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity).toHaveProperty('userEmail');
           expect(activity.userEmail).toBe('testuser@example.com');
         });
@@ -421,9 +421,9 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities.length).toBeGreaterThan(0);
+        expect(res.body.activities.length).toBeGreaterThan(0);
         // userName and userEmail should be null for deleted users
-        const activity = res.body.data.activities[0];
+        const activity = res.body.activities[0];
         expect(activity.userName).toBeNull();
         expect(activity.userEmail).toBeNull();
       });
@@ -450,7 +450,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        const timestamps = res.body.data.activities.map(a => new Date(a.timestamp).getTime());
+        const timestamps = res.body.activities.map(a => new Date(a.timestamp).getTime());
         for (let i = 1; i < timestamps.length; i++) {
           expect(timestamps[i - 1]).toBeGreaterThanOrEqual(timestamps[i]);
         }
@@ -569,7 +569,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data).toHaveProperty('activities');
+        expect(res.body).toHaveProperty('activities');
       });
     });
 
@@ -596,8 +596,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(15);
+        expect(res.body.activities).toHaveLength(15);
-        res.body.data.activities.forEach(activity => {
+        res.body.activities.forEach(activity => {
           expect(activity.userId.toString()).toBe(user1._id.toString());
         });
       });
@@ -647,8 +647,8 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(5);
+        expect(res.body.activities).toHaveLength(5);
-        expect(res.body.data.pagination).toEqual(expect.objectContaining({
+        expect(res.body.pagination).toEqual(expect.objectContaining({
           page: 1,
           limit: 5,
           total: 15,
@@ -668,7 +668,7 @@ describe('Phase 2: Activity Monitoring API', () => {
           .set('Cookie', [`auth_token=${token}`])
           .expect(200);
 
-        expect(res.body.data.activities).toHaveLength(0);
+        expect(res.body.activities).toHaveLength(0);
       });
     });
   });

__tests__/phase4-settings-security.test.js

@@ -255,8 +255,8 @@ describe('Phase 4: Settings & Security', () => {
           .send({ activityRetentionDays: 60 })
           .expect(200);
 
-        expect(response.body.data).toHaveProperty('settings');
+        expect(response.body).toHaveProperty('settings');
-        expect(response.body.data).toHaveProperty('message');
+        expect(response.body).toHaveProperty('message');
       });
     });
 
@@ -287,7 +287,7 @@ describe('Phase 4: Settings & Security', () => {
           .send({ activityRetentionDays: 30 })
           .expect(200);
 
-        expect(response.body.data.settings.activityRetentionDays).toBe(30);
+        expect(response.body.settings.activityRetentionDays).toBe(30);
       });
 
       it('should accept retention period at maximum boundary (365 days)', async () => {
@@ -296,7 +296,7 @@ describe('Phase 4: Settings & Security', () => {
           .send({ activityRetentionDays: 365 })
           .expect(200);
 
-        expect(response.body.data.settings.activityRetentionDays).toBe(365);
+        expect(response.body.settings.activityRetentionDays).toBe(365);
       });
 
       it('should accept valid retention period within range (180 days)', async () => {
@@ -305,7 +305,7 @@ describe('Phase 4: Settings & Security', () => {
           .send({ activityRetentionDays: 180 })
           .expect(200);
 
-        expect(response.body.data.settings.activityRetentionDays).toBe(180);
+        expect(response.body.settings.activityRetentionDays).toBe(180);
       });
 
       it('should return 400 for non-numeric retention period', async () => {
@@ -343,8 +343,8 @@ describe('Phase 4: Settings & Security', () => {
           .send({ activityRetentionDays: 120 })
           .expect(200);
 
-        expect(response.body.data.settings.activityRetentionDays).toBe(120);
+        expect(response.body.settings.activityRetentionDays).toBe(120);
-        expect(response.body.data.message).toBe('Settings updated successfully');
+        expect(response.body.message).toBe('Settings updated successfully');
       });
 
       it('should persist the updated settings in database', async () => {
@@ -643,7 +643,7 @@ describe('Phase 4: Settings & Security', () => {
         .send({ role: 'user' })
         .expect(200);
 
-      expect(response.body.data.user.role).toBe('user');
+      expect(response.body.user.role).toBe('user');
     });
 
     it('should not allow self-demotion even when other admins exist', async () => {
@@ -688,7 +688,7 @@ describe('Phase 4: Settings & Security', () => {
         .send({ role: 'user' })
         .expect(200);
 
-      expect(response1.body.data.user.role).toBe('user');
+      expect(response1.body.user.role).toBe('user');
 
       // Now the primary admin is the last one, cannot demote
       const response2 = await authAs(testUsers.admin)
@@ -702,9 +702,8 @@ describe('Phase 4: Settings & Security', () => {
 
   // ============================================================
   // SEC-4.3: Rate Limiting on Admin Endpoints
-  // SKIPPED: Rate limiting moved to Phase 5
   // ============================================================
-  describe.skip('SEC-4.3: Rate Limiting', () => {
+  describe('SEC-4.3: Rate Limiting', () => {
     /**
      * Helper to make multiple rapid requests
      * @param {string} endpoint - API endpoint
@@ -918,8 +917,8 @@ describe('Integration: Settings Update Workflow', () => {
       .send({ activityRetentionDays: 180 })
       .expect(200);
 
-    expect(updateResponse.body.data.settings.activityRetentionDays).toBe(180);
+    expect(updateResponse.body.settings.activityRetentionDays).toBe(180);
-    expect(updateResponse.body.data.message).toBe('Settings updated successfully');
+    expect(updateResponse.body.message).toBe('Settings updated successfully');
 
     // Step 3: Verify settings persisted
     const verifyResponse = await authAdmin('get', '/api/admin/settings')
@@ -963,7 +962,7 @@ describe('Integration: Settings Update Workflow', () => {
         .send({ activityRetentionDays: value })
         .expect(200);
 
-      expect(response.body.data.settings.activityRetentionDays).toBe(value);
+      expect(response.body.settings.activityRetentionDays).toBe(value);
 
       // Verify TTL index after each update
       const indexes = await db.collection('user_activity').indexes();

__tests__/scraper/dataConversion.test.js

@@ -1,496 +0,0 @@
-/**
- * Tests for convertDataTypes() and its internal helper functions
- *
- * Covers:
- * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
- * - parseInteger: valid integers, null/empty/invalid
- * - parsePositiveInteger: treats 0 as null (for area)
- * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
- * - parseFloat: "1.5" for half baths, null/empty/invalid
- * - parseBoolean: "true"/"false"/"1"/"0", null/empty
- * - trimString: trim whitespace, null for empty, null/undefined
- * - convertDataTypes: full integration applying correct parser to each field
- */
-
-const {
-  convertDataTypes,
-  parseInteger,
-  parsePositiveInteger,
-  parseIntegerOrString,
-  parsePrice,
-  parseBoolean,
-  parseFloatValue,
-  trimString
-} = require('../../services/scraperService');
-
-// ---------------------------------------------------------------
-// 1. parsePrice
-// ---------------------------------------------------------------
-describe('parsePrice', () => {
-  it('should parse "$1,234" format to 1234', () => {
-    expect(parsePrice('$1,234')).toBe(1234);
-  });
-
-  it('should parse plain numeric string', () => {
-    expect(parsePrice('1450')).toBe(1450);
-  });
-
-  it('should parse "$2,500.00" stripping non-numeric chars', () => {
-    expect(parsePrice('$2,500.00')).toBe(2500);
-  });
-
-  it('should return null for "Call for pricing"', () => {
-    expect(parsePrice('Call for pricing')).toBeNull();
-  });
-
-  it('should return null for "Contact us"', () => {
-    expect(parsePrice('Contact us')).toBeNull();
-  });
-
-  it('should return null for "Please inquire"', () => {
-    expect(parsePrice('Please inquire')).toBeNull();
-  });
-
-  it('should return null for negative values', () => {
-    // After stripping non-numeric chars, "-1500" becomes "1500"
-    // but if someone passes a numeric -1500 directly, check that too
-    expect(parsePrice('-')).toBeNull();
-  });
-
-  it('should return null for null', () => {
-    expect(parsePrice(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parsePrice('')).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(parsePrice(undefined)).toBeNull();
-  });
-
-  it('should return null for completely non-numeric string', () => {
-    expect(parsePrice('abc')).toBeNull();
-  });
-
-  it('should handle price of 0', () => {
-    expect(parsePrice('0')).toBe(0);
-  });
-
-  it('should handle large prices', () => {
-    expect(parsePrice('$15,500')).toBe(15500);
-  });
-});
-
-// ---------------------------------------------------------------
-// 2. parseInteger
-// ---------------------------------------------------------------
-describe('parseInteger', () => {
-  it('should parse valid integer string', () => {
-    expect(parseInteger('42')).toBe(42);
-  });
-
-  it('should parse "0" to 0', () => {
-    expect(parseInteger('0')).toBe(0);
-  });
-
-  it('should parse negative integer string', () => {
-    expect(parseInteger('-5')).toBe(-5);
-  });
-
-  it('should return null for null', () => {
-    expect(parseInteger(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parseInteger('')).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(parseInteger(undefined)).toBeNull();
-  });
-
-  it('should return null for non-numeric string', () => {
-    expect(parseInteger('abc')).toBeNull();
-  });
-
-  it('should return null for NaN', () => {
-    expect(parseInteger('NaN')).toBeNull();
-  });
-
-  it('should return null for Infinity', () => {
-    expect(parseInteger('Infinity')).toBeNull();
-  });
-
-  it('should truncate float strings to integer', () => {
-    expect(parseInteger('3.7')).toBe(3);
-  });
-});
-
-// ---------------------------------------------------------------
-// 3. parsePositiveInteger
-// ---------------------------------------------------------------
-describe('parsePositiveInteger', () => {
-  it('should parse positive integer normally', () => {
-    expect(parsePositiveInteger('750')).toBe(750);
-  });
-
-  it('should treat 0 as null (for area field)', () => {
-    expect(parsePositiveInteger('0')).toBeNull();
-  });
-
-  it('should return null for null input', () => {
-    expect(parsePositiveInteger(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parsePositiveInteger('')).toBeNull();
-  });
-
-  it('should return null for non-numeric string', () => {
-    expect(parsePositiveInteger('abc')).toBeNull();
-  });
-
-  it('should handle negative values (parsed as negative int, not positive)', () => {
-    expect(parsePositiveInteger('-5')).toBe(-5);
-  });
-});
-
-// ---------------------------------------------------------------
-// 4. parseIntegerOrString
-// ---------------------------------------------------------------
-describe('parseIntegerOrString', () => {
-  it('should return integer when value is cleanly parseable', () => {
-    expect(parseIntegerOrString('5001')).toBe(5001);
-  });
-
-  it('should return trimmed string for non-numeric value', () => {
-    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
-  });
-
-  it('should return trimmed string when value has non-numeric suffix', () => {
-    expect(parseIntegerOrString('123abc')).toBe('123abc');
-  });
-
-  it('should return null for null', () => {
-    expect(parseIntegerOrString(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parseIntegerOrString('')).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(parseIntegerOrString(undefined)).toBeNull();
-  });
-
-  it('should trim whitespace before checking', () => {
-    expect(parseIntegerOrString('  5001  ')).toBe(5001);
-  });
-
-  it('should return trimmed string for non-numeric with whitespace', () => {
-    expect(parseIntegerOrString('  abc  ')).toBe('abc');
-  });
-});
-
-// ---------------------------------------------------------------
-// 5. parseFloatValue
-// ---------------------------------------------------------------
-describe('parseFloatValue', () => {
-  it('should parse "1.5" for half baths', () => {
-    expect(parseFloatValue('1.5')).toBe(1.5);
-  });
-
-  it('should parse "2.0" to 2', () => {
-    expect(parseFloatValue('2.0')).toBe(2.0);
-  });
-
-  it('should parse integer string as float', () => {
-    expect(parseFloatValue('3')).toBe(3);
-  });
-
-  it('should return null for null', () => {
-    expect(parseFloatValue(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parseFloatValue('')).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(parseFloatValue(undefined)).toBeNull();
-  });
-
-  it('should return null for non-numeric string', () => {
-    expect(parseFloatValue('abc')).toBeNull();
-  });
-
-  it('should return null for NaN', () => {
-    expect(parseFloatValue('NaN')).toBeNull();
-  });
-
-  it('should return null for Infinity', () => {
-    expect(parseFloatValue('Infinity')).toBeNull();
-  });
-});
-
-// ---------------------------------------------------------------
-// 6. parseBoolean
-// ---------------------------------------------------------------
-describe('parseBoolean', () => {
-  it('should parse "true" to true', () => {
-    expect(parseBoolean('true')).toBe(true);
-  });
-
-  it('should parse "false" to false', () => {
-    expect(parseBoolean('false')).toBe(false);
-  });
-
-  it('should parse "1" to true', () => {
-    expect(parseBoolean('1')).toBe(true);
-  });
-
-  it('should parse "0" to false', () => {
-    expect(parseBoolean('0')).toBe(false);
-  });
-
-  it('should parse "TRUE" (case insensitive) to true', () => {
-    expect(parseBoolean('TRUE')).toBe(true);
-  });
-
-  it('should parse "False" (case insensitive) to false', () => {
-    expect(parseBoolean('False')).toBe(false);
-  });
-
-  it('should pass through boolean true', () => {
-    expect(parseBoolean(true)).toBe(true);
-  });
-
-  it('should pass through boolean false', () => {
-    expect(parseBoolean(false)).toBe(false);
-  });
-
-  it('should return null for null', () => {
-    expect(parseBoolean(null)).toBeNull();
-  });
-
-  it('should return null for empty string', () => {
-    expect(parseBoolean('')).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(parseBoolean(undefined)).toBeNull();
-  });
-
-  it('should return null for unrecognized string', () => {
-    expect(parseBoolean('yes')).toBeNull();
-  });
-});
-
-// ---------------------------------------------------------------
-// 7. trimString
-// ---------------------------------------------------------------
-describe('trimString', () => {
-  it('should trim leading and trailing whitespace', () => {
-    expect(trimString('  hello  ')).toBe('hello');
-  });
-
-  it('should return the string as-is if no whitespace', () => {
-    expect(trimString('hello')).toBe('hello');
-  });
-
-  it('should return null for empty string', () => {
-    expect(trimString('')).toBeNull();
-  });
-
-  it('should return null for whitespace-only string', () => {
-    expect(trimString('   ')).toBeNull();
-  });
-
-  it('should return null for null', () => {
-    expect(trimString(null)).toBeNull();
-  });
-
-  it('should return null for undefined', () => {
-    expect(trimString(undefined)).toBeNull();
-  });
-
-  it('should convert non-string to string then trim', () => {
-    expect(trimString(123)).toBe('123');
-  });
-});
-
-// ---------------------------------------------------------------
-// 8. convertDataTypes - Full integration test
-// ---------------------------------------------------------------
-describe('convertDataTypes', () => {
-  it('should apply correct parser to each field', () => {
-    const rawUnit = {
-      id: '1001',
-      unit_code: 'CCT-101',
-      unit_id: '5001',
-      floor: '1',
-      area: '750',
-      bed_count: '1',
-      bath_count: '1.5',
-      price: '1450',
-      available: 'true',
-      unavailable: 'false',
-      soonest: '2026-03-01',
-      date_available: '20260301',
-      plan_id: '201',
-      plan_name: 'Alpine',
-      obj_type: 'unit',
-      community: 'Country Club Towers',
-      asset: '100',
-      href: '/units/CCT-101',
-      inventory_href: '/inventory/CCT-101',
-      image_url: 'https://example.com/images/unit-101.jpg',
-      specials_content: 'First month free!'
-    };
-
-    const converted = convertDataTypes(rawUnit);
-
-    // Integer fields
-    expect(converted.id).toBe(1001);
-    expect(converted.floor).toBe(1);
-    expect(converted.bed_count).toBe(1);
-    expect(converted.plan_id).toBe(201);
-    expect(converted.asset).toBe(100);
-    expect(converted.date_available).toBe(20260301);
-
-    // parseIntegerOrString
-    expect(converted.unit_id).toBe(5001);
-
-    // parsePositiveInteger (area)
-    expect(converted.area).toBe(750);
-
-    // Float field (bath_count)
-    expect(converted.bath_count).toBe(1.5);
-
-    // Price field
-    expect(converted.price).toBe(1450);
-
-    // Boolean fields
-    expect(converted.available).toBe(true);
-    expect(converted.unavailable).toBe(false);
-
-    // String fields (trimmed)
-    expect(converted.unit_code).toBe('CCT-101');
-    expect(converted.plan_name).toBe('Alpine');
-    expect(converted.soonest).toBe('2026-03-01');
-    expect(converted.obj_type).toBe('unit');
-    expect(converted.community).toBe('Country Club Towers');
-    expect(converted.href).toBe('/units/CCT-101');
-    expect(converted.inventory_href).toBe('/inventory/CCT-101');
-    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
-    expect(converted.specials_content).toBe('First month free!');
-  });
-
-  it('should preserve string fields without converting them', () => {
-    const rawUnit = {
-      unit_code: 'CCT-205',
-      plan_name: '  Birch  ',
-      soonest: 'Now',
-      obj_type: 'unit',
-      community: '  Country Club Towers  ',
-      href: '/units/CCT-205',
-      inventory_href: '/inventory/CCT-205',
-      image_url: 'https://example.com/img.jpg',
-      specials_content: null
-    };
-
-    const converted = convertDataTypes(rawUnit);
-
-    expect(converted.unit_code).toBe('CCT-205');
-    expect(converted.plan_name).toBe('Birch');
-    expect(converted.community).toBe('Country Club Towers');
-    expect(converted.specials_content).toBeNull();
-  });
-
-  it('should handle a unit with all null/undefined/empty values', () => {
-    const rawUnit = {
-      id: null,
-      unit_code: undefined,
-      unit_id: '',
-      floor: null,
-      area: '',
-      bed_count: undefined,
-      bath_count: null,
-      price: '',
-      available: null,
-      unavailable: undefined,
-      soonest: '',
-      date_available: null,
-      plan_id: undefined,
-      plan_name: '',
-      obj_type: null,
-      community: undefined,
-      asset: '',
-      href: null,
-      inventory_href: undefined,
-      image_url: '',
-      specials_content: null
-    };
-
-    const converted = convertDataTypes(rawUnit);
-
-    expect(converted.id).toBeNull();
-    expect(converted.unit_code).toBeNull();
-    expect(converted.unit_id).toBeNull();
-    expect(converted.floor).toBeNull();
-    expect(converted.area).toBeNull();
-    expect(converted.bed_count).toBeNull();
-    expect(converted.bath_count).toBeNull();
-    expect(converted.price).toBeNull();
-    expect(converted.available).toBeNull();
-    expect(converted.unavailable).toBeNull();
-    expect(converted.soonest).toBeNull();
-    expect(converted.date_available).toBeNull();
-    expect(converted.plan_id).toBeNull();
-    expect(converted.plan_name).toBeNull();
-    expect(converted.obj_type).toBeNull();
-    expect(converted.community).toBeNull();
-    expect(converted.asset).toBeNull();
-    expect(converted.href).toBeNull();
-    expect(converted.inventory_href).toBeNull();
-    expect(converted.image_url).toBeNull();
-    expect(converted.specials_content).toBeNull();
-  });
-
-  it('should handle area=0 as null (parsePositiveInteger)', () => {
-    const rawUnit = { area: '0' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.area).toBeNull();
-  });
-
-  it('should handle bed_count=0 as valid (studio)', () => {
-    const rawUnit = { bed_count: '0' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.bed_count).toBe(0);
-  });
-
-  it('should handle "Call for pricing" in price field', () => {
-    const rawUnit = { price: 'Call for pricing' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.price).toBeNull();
-  });
-
-  it('should handle "$1,234" price format', () => {
-    const rawUnit = { price: '$1,234' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.price).toBe(1234);
-  });
-
-  it('should handle unit_id as string identifier', () => {
-    const rawUnit = { unit_id: 'UNIT-ABC' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.unit_id).toBe('UNIT-ABC');
-  });
-
-  it('should handle bath_count with half baths', () => {
-    const rawUnit = { bath_count: '1.5' };
-    const converted = convertDataTypes(rawUnit);
-    expect(converted.bath_count).toBe(1.5);
-  });
-});

__tests__/scraper/fetchPage.test.js

@@ -1,535 +0,0 @@
-/**
- * Tests for fetchPage function
- *
- * Acceptance Criteria:
- * - Uses axios for HTTP GET
- * - Enforces timeout from config (default 30s)
- * - Sets User-Agent header from config
- * - Implements retry with exponential backoff (1s, 2s, 4s)
- * - Retries on 5xx errors and network timeouts
- * - Does NOT retry on 4xx errors
- * - Logs each attempt with attempt number
- * - Returns HTML string on success
- * - Throws error after all retries exhausted
- */
-
-const axios = require('axios');
-
-// Mock axios
-jest.mock('axios');
-
-// Mock config
-jest.mock('../../config/scraper', () => ({
-  RETRY_CONFIG: {
-    maxRetries: 3,
-    baseDelay: 1000,
-    timeout: 30000
-  },
-  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
-}));
-
-// Import after mocking
-const { fetchPage } = require('../../services/scraperService');
-const config = require('../../config/scraper');
-
-describe('fetchPage', () => {
-  let mockLogger;
-
-  beforeEach(() => {
-    jest.clearAllMocks();
-    jest.useFakeTimers();
-
-    // Create mock logger
-    mockLogger = {
-      info: jest.fn(),
-      warn: jest.fn(),
-      error: jest.fn()
-    };
-  });
-
-  afterEach(() => {
-    jest.useRealTimers();
-  });
-
-  describe('successful requests', () => {
-    it('should return HTML string on success', async () => {
-      const htmlContent = '<html><body>Test content</body></html>';
-
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: htmlContent
-      });
-
-      const result = await fetchPage('https://example.com', mockLogger);
-
-      expect(result).toBe(htmlContent);
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-
-    it('should use axios for HTTP GET', async () => {
-      const url = 'https://example.com/apartments';
-
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage(url, mockLogger);
-
-      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
-    });
-
-    it('should set timeout from config', async () => {
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      expect(axios.get).toHaveBeenCalledWith(
-        expect.any(String),
-        expect.objectContaining({
-          timeout: config.RETRY_CONFIG.timeout
-        })
-      );
-    });
-
-    it('should set User-Agent header from config', async () => {
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      expect(axios.get).toHaveBeenCalledWith(
-        expect.any(String),
-        expect.objectContaining({
-          headers: expect.objectContaining({
-            'User-Agent': config.USER_AGENT
-          })
-        })
-      );
-    });
-
-    it('should log the attempt with attempt number', async () => {
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Fetching page',
-        expect.objectContaining({
-          url: 'https://example.com',
-          attempt: 1
-        })
-      );
-    });
-
-    it('should log successful fetch with status and content length', async () => {
-      const htmlContent = '<html><body>Content</body></html>';
-
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: htmlContent
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Page fetched successfully',
-        expect.objectContaining({
-          status: 200,
-          contentLength: htmlContent.length
-        })
-      );
-    });
-  });
-
-  describe('retry behavior on 5xx errors', () => {
-    it('should retry on 500 Internal Server Error', async () => {
-      const error500 = new Error('Internal Server Error');
-      error500.response = { status: 500 };
-
-      axios.get
-        .mockRejectedValueOnce(error500)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-
-      // Fast-forward through the retry delay
-      await jest.advanceTimersByTimeAsync(1000);
-
-      const result = await promise;
-
-      expect(result).toBe('<html></html>');
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-
-    it('should retry on 502 Bad Gateway', async () => {
-      const error502 = new Error('Bad Gateway');
-      error502.response = { status: 502 };
-
-      axios.get
-        .mockRejectedValueOnce(error502)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-
-    it('should retry on 503 Service Unavailable', async () => {
-      const error503 = new Error('Service Unavailable');
-      error503.response = { status: 503 };
-
-      axios.get
-        .mockRejectedValueOnce(error503)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-
-    it('should retry on 504 Gateway Timeout', async () => {
-      const error504 = new Error('Gateway Timeout');
-      error504.response = { status: 504 };
-
-      axios.get
-        .mockRejectedValueOnce(error504)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-  });
-
-  describe('retry behavior on network errors', () => {
-    it('should retry on network timeout (ECONNABORTED)', async () => {
-      const timeoutError = new Error('timeout of 30000ms exceeded');
-      timeoutError.code = 'ECONNABORTED';
-      // Network errors don't have a response property
-
-      axios.get
-        .mockRejectedValueOnce(timeoutError)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      const result = await promise;
-
-      expect(result).toBe('<html></html>');
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-
-    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
-      const dnsError = new Error('getaddrinfo ENOTFOUND');
-      dnsError.code = 'ENOTFOUND';
-
-      axios.get
-        .mockRejectedValueOnce(dnsError)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-
-    it('should retry on connection refused (ECONNREFUSED)', async () => {
-      const connError = new Error('connect ECONNREFUSED');
-      connError.code = 'ECONNREFUSED';
-
-      axios.get
-        .mockRejectedValueOnce(connError)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(2);
-    });
-  });
-
-  describe('no retry on 4xx errors', () => {
-    it('should NOT retry on 400 Bad Request', async () => {
-      const error400 = new Error('Bad Request');
-      error400.response = { status: 400 };
-
-      axios.get.mockRejectedValueOnce(error400);
-
-      await expect(fetchPage('https://example.com', mockLogger))
-        .rejects.toThrow();
-
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-
-    it('should NOT retry on 401 Unauthorized', async () => {
-      const error401 = new Error('Unauthorized');
-      error401.response = { status: 401 };
-
-      axios.get.mockRejectedValueOnce(error401);
-
-      await expect(fetchPage('https://example.com', mockLogger))
-        .rejects.toThrow();
-
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-
-    it('should NOT retry on 403 Forbidden', async () => {
-      const error403 = new Error('Forbidden');
-      error403.response = { status: 403 };
-
-      axios.get.mockRejectedValueOnce(error403);
-
-      await expect(fetchPage('https://example.com', mockLogger))
-        .rejects.toThrow();
-
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-
-    it('should NOT retry on 404 Not Found', async () => {
-      const error404 = new Error('Not Found');
-      error404.response = { status: 404 };
-
-      axios.get.mockRejectedValueOnce(error404);
-
-      await expect(fetchPage('https://example.com', mockLogger))
-        .rejects.toThrow();
-
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-
-    it('should NOT retry on 429 Too Many Requests', async () => {
-      const error429 = new Error('Too Many Requests');
-      error429.response = { status: 429 };
-
-      axios.get.mockRejectedValueOnce(error429);
-
-      await expect(fetchPage('https://example.com', mockLogger))
-        .rejects.toThrow();
-
-      expect(axios.get).toHaveBeenCalledTimes(1);
-    });
-  });
-
-  describe('exponential backoff', () => {
-    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
-      const error500 = new Error('Internal Server Error');
-      error500.response = { status: 500 };
-
-      // Fail 3 times, then succeed
-      axios.get
-        .mockRejectedValueOnce(error500)
-        .mockRejectedValueOnce(error500)
-        .mockRejectedValueOnce(error500)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-
-      // First retry: 1 second delay
-      await jest.advanceTimersByTimeAsync(1000);
-
-      // Second retry: 2 second delay
-      await jest.advanceTimersByTimeAsync(2000);
-
-      // Third retry: 4 second delay
-      await jest.advanceTimersByTimeAsync(4000);
-
-      await promise;
-
-      expect(axios.get).toHaveBeenCalledTimes(4);
-    });
-
-    it('should log each retry attempt with attempt number', async () => {
-      const error500 = new Error('Internal Server Error');
-      error500.response = { status: 500 };
-
-      axios.get
-        .mockRejectedValueOnce(error500)
-        .mockRejectedValueOnce(error500)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-
-      await jest.advanceTimersByTimeAsync(1000);
-      await jest.advanceTimersByTimeAsync(2000);
-
-      await promise;
-
-      // Check that attempt numbers were logged
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Fetching page',
-        expect.objectContaining({ attempt: 1 })
-      );
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Fetching page',
-        expect.objectContaining({ attempt: 2 })
-      );
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Fetching page',
-        expect.objectContaining({ attempt: 3 })
-      );
-    });
-
-    it('should log failed attempts with error details', async () => {
-      const error500 = new Error('Internal Server Error');
-      error500.response = { status: 500 };
-
-      axios.get
-        .mockRejectedValueOnce(error500)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(mockLogger.warn).toHaveBeenCalledWith(
-        'Fetch attempt failed',
-        expect.objectContaining({
-          attempt: 1,
-          statusCode: 500,
-          isRetryable: true
-        })
-      );
-    });
-
-    it('should log wait time before retry', async () => {
-      const error500 = new Error('Internal Server Error');
-      error500.response = { status: 500 };
-
-      axios.get
-        .mockRejectedValueOnce(error500)
-        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
-
-      const promise = fetchPage('https://example.com', mockLogger);
-      await jest.advanceTimersByTimeAsync(1000);
-
-      await promise;
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Waiting before retry',
-        expect.objectContaining({ delay: 1000 })
-      );
-    });
-  });
-
-  describe('retry exhaustion', () => {
-    // Use real timers for exhaustion tests to avoid promise handling issues
-    beforeEach(() => {
-      jest.useRealTimers();
-    });
-
-    afterEach(() => {
-      jest.useFakeTimers();
-    });
-
-    it('should throw error after all retries exhausted (4 attempts total)', async () => {
-      // Override config for faster tests
-      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
-      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
-
-      try {
-        const error500 = new Error('Internal Server Error');
-        error500.response = { status: 500 };
-
-        // All 4 attempts fail (1 initial + 3 retries)
-        axios.get
-          .mockRejectedValueOnce(error500)
-          .mockRejectedValueOnce(error500)
-          .mockRejectedValueOnce(error500)
-          .mockRejectedValueOnce(error500);
-
-        await expect(fetchPage('https://example.com', mockLogger))
-          .rejects.toThrow('Internal Server Error');
-
-        expect(axios.get).toHaveBeenCalledTimes(4);
-      } finally {
-        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
-      }
-    });
-
-    it('should throw the last error when retries are exhausted', async () => {
-      // Override config for faster tests
-      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
-      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
-
-      try {
-        const error500 = new Error('Server Error');
-        error500.response = { status: 500 };
-
-        axios.get.mockRejectedValue(error500);
-
-        await expect(fetchPage('https://example.com', mockLogger))
-          .rejects.toBe(error500);
-      } finally {
-        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
-      }
-    });
-  });
-
-  describe('axios configuration', () => {
-    it('should set maxRedirects to 5', async () => {
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      expect(axios.get).toHaveBeenCalledWith(
-        expect.any(String),
-        expect.objectContaining({
-          maxRedirects: 5
-        })
-      );
-    });
-
-    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
-      axios.get.mockResolvedValueOnce({
-        status: 200,
-        data: '<html></html>'
-      });
-
-      await fetchPage('https://example.com', mockLogger);
-
-      const callArgs = axios.get.mock.calls[0][1];
-      const validateStatus = callArgs.validateStatus;
-
-      // Should accept 2xx
-      expect(validateStatus(200)).toBe(true);
-      expect(validateStatus(201)).toBe(true);
-      expect(validateStatus(204)).toBe(true);
-
-      // Should accept 3xx
-      expect(validateStatus(301)).toBe(true);
-      expect(validateStatus(302)).toBe(true);
-
-      // Should reject 4xx and 5xx
-      expect(validateStatus(400)).toBe(false);
-      expect(validateStatus(404)).toBe(false);
-      expect(validateStatus(500)).toBe(false);
-    });
-  });
-});

__tests__/scraper/fixtures/sample-listing-call.html

@@ -1,64 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head><title>Country Club Towers - Call for Pricing</title></head>
-<body>
-<section class="spaces__tab-unit">
-  <article
-    class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
-    data-spaces-id="9001"
-    data-spaces-soonest="Now"
-    data-spaces-sort-date="1706745600"
-    data-spaces-sort-price="Call for pricing"
-    data-spaces-sort-area="750"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="P-101"
-    data-spaces-unit-id="9001"
-    data-spaces-unit-floor="1"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="500"
-    data-spaces-sort-plan-name="Penthouse A"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=9001"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
-    aria-label="Unit P-101">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">Call for pricing</span>
-      <span class="spaces-unit__unit">P-101</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
-    data-spaces-id="9002"
-    data-spaces-soonest="Now"
-    data-spaces-sort-date="1706745600"
-    data-spaces-sort-price="Call for pricing"
-    data-spaces-sort-area="900"
-    data-spaces-sort-bed="2"
-    data-spaces-unit="P-102"
-    data-spaces-unit-id="9002"
-    data-spaces-unit-floor="2"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="501"
-    data-spaces-sort-plan-name="Penthouse B"
-    data-spaces-sort-bath="2"
-    data-spaces-href="?spaces_tab=unit-detail&detail=9002"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
-    aria-label="Unit P-102">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">Call for pricing</span>
-      <span class="spaces-unit__unit">P-102</span>
-    </div>
-  </article>
-</section>
-</body>
-</html>

__tests__/scraper/fixtures/sample-listing-empty.html

@@ -1,9 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head><title>Country Club Towers - No Units</title></head>
-<body>
-<section class="spaces__tab-unit">
-  <!-- Empty - no units available -->
-</section>
-</body>
-</html>

__tests__/scraper/fixtures/sample-listing.html

@@ -1,288 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head><title>Country Club Towers - Apartments</title></head>
-<body>
-<section class="spaces__tab-unit">
-  <article
-    class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154842"
-    data-spaces-soonest="2025-10-11"
-    data-spaces-sort-date="1760140800"
-    data-spaces-sort-price="5230"
-    data-spaces-sort-area="1210"
-    data-spaces-sort-bed="2"
-    data-spaces-unit="W2707"
-    data-spaces-unit-id="154842"
-    data-spaces-unit-floor="2755"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="10270"
-    data-spaces-sort-plan-name="Pyramid Peak - Terrace"
-    data-spaces-sort-bath="2"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154842"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
-    aria-label="Unit W2707">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">5230</span>
-      <span class="spaces-unit__unit">W2707</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154451"
-    data-spaces-soonest="2026-03-08"
-    data-spaces-sort-date="1772928000"
-    data-spaces-sort-price="2767"
-    data-spaces-sort-area="806"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="E0901"
-    data-spaces-unit-id="154451"
-    data-spaces-unit-floor="2738"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8016"
-    data-spaces-sort-plan-name="Sunshine Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154451"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
-    aria-label="Unit E0901">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2767</span>
-      <span class="spaces-unit__unit">E0901</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154814"
-    data-spaces-soonest="2025-12-18"
-    data-spaces-sort-date="1766016000"
-    data-spaces-sort-price="2255"
-    data-spaces-sort-area="623"
-    data-spaces-sort-bed="0"
-    data-spaces-unit="W2603"
-    data-spaces-unit-id="154814"
-    data-spaces-unit-floor="2754"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8018"
-    data-spaces-sort-plan-name="Little Bear Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154814"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
-    aria-label="Unit W2603">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2255</span>
-      <span class="spaces-unit__unit">W2603</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154428"
-    data-spaces-soonest="2026-02-22"
-    data-spaces-sort-date="1771718400"
-    data-spaces-sort-price="4250"
-    data-spaces-sort-area="1152"
-    data-spaces-sort-bed="2"
-    data-spaces-unit="W0802"
-    data-spaces-unit-id="154428"
-    data-spaces-unit-floor="2737"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8017"
-    data-spaces-sort-plan-name="Mt. Princeton"
-    data-spaces-sort-bath="2"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154428"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
-    aria-label="Unit W0802">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">4250</span>
-      <span class="spaces-unit__unit">W0802</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154778"
-    data-spaces-soonest="2025-12-31"
-    data-spaces-sort-date="1767139200"
-    data-spaces-sort-price="2495"
-    data-spaces-sort-area="764"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="W2405"
-    data-spaces-unit-id="154778"
-    data-spaces-unit-floor="2752"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8020"
-    data-spaces-sort-plan-name="Grays Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154778"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
-    aria-label="Unit W2405">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2495</span>
-      <span class="spaces-unit__unit">W2405</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154442"
-    data-spaces-soonest="2025-02-01"
-    data-spaces-sort-date="1738368000"
-    data-spaces-sort-price="2495"
-    data-spaces-sort-area="802"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="W0809"
-    data-spaces-unit-id="154442"
-    data-spaces-unit-floor="2737"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8022"
-    data-spaces-sort-plan-name="Pikes Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154442"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
-    aria-label="Unit W0809">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2495</span>
-      <span class="spaces-unit__unit">W0809</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154764"
-    data-spaces-soonest="2026-02-17"
-    data-spaces-sort-date="1771286400"
-    data-spaces-sort-price="2435"
-    data-spaces-sort-area="715"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="W2308"
-    data-spaces-unit-id="154764"
-    data-spaces-unit-floor="2751"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8023"
-    data-spaces-sort-plan-name="Longs Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154764"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
-    aria-label="Unit W2308">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2435</span>
-      <span class="spaces-unit__unit">W2308</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154612"
-    data-spaces-soonest="2026-03-24"
-    data-spaces-sort-date="1774310400"
-    data-spaces-sort-price="2683"
-    data-spaces-sort-area="797"
-    data-spaces-sort-bed="1"
-    data-spaces-unit="W1610"
-    data-spaces-unit-id="154612"
-    data-spaces-unit-floor="2744"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8024"
-    data-spaces-sort-plan-name="Torreys Peak"
-    data-spaces-sort-bath="1"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154612"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
-    aria-label="Unit W1610">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">2683</span>
-      <span class="spaces-unit__unit">W1610</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
-    data-spaces-id="154495"
-    data-spaces-soonest="2025-11-07"
-    data-spaces-sort-date="1762473600"
-    data-spaces-sort-price="4470"
-    data-spaces-sort-area="1230"
-    data-spaces-sort-bed="2"
-    data-spaces-unit="E1011"
-    data-spaces-unit-id="154495"
-    data-spaces-unit-floor="2739"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8025"
-    data-spaces-sort-plan-name="Maroon Peak"
-    data-spaces-sort-bath="2"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154495"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
-    aria-label="Unit E1011">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">4470</span>
-      <span class="spaces-unit__unit">E1011</span>
-    </div>
-  </article>
-  <article
-    class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
-    data-spaces-id="154919"
-    data-spaces-soonest="2025-10-27"
-    data-spaces-sort-date="1761523200"
-    data-spaces-sort-price="8581"
-    data-spaces-sort-area="1532"
-    data-spaces-sort-bed="2"
-    data-spaces-unit="E3205"
-    data-spaces-unit-id="154919"
-    data-spaces-unit-floor="2760"
-    data-spaces-obj="unit"
-    data-spaces-unavailable="false"
-    data-spaces-available="true"
-    data-spaces-community="Country Club Towers"
-    data-spaces-asset="420"
-    data-spaces-specials-content=""
-    data-spaces-plan-id="8037"
-    data-spaces-sort-plan-name="Mt. Antero"
-    data-spaces-sort-bath="2.5"
-    data-spaces-href="?spaces_tab=unit-detail&detail=154919"
-    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
-    aria-label="Unit E3205">
-    <div class="spaces-unit__inner">
-      <span class="spaces-unit__price">8581</span>
-      <span class="spaces-unit__unit">E3205</span>
-    </div>
-  </article>
-</section>
-</body>
-</html>

__tests__/scraper/insertPrices.test.js

@@ -1,582 +0,0 @@
-/**
- * Tests for insertPrices() bulk operation
- *
- * Covers:
- * - Only units with non-null prices are inserted
- * - bulkWrite uses updateOne with upsert for idempotency
- * - Filter uses unit_code + date_checked composite key
- * - Price record includes unit_code, date_checked, price, last_updated, data_source
- * - Empty price records logs warning and returns early
- * - Error handling when bulkWrite fails
- * - Logging of inserted and updated counts
- * - Re-running on same day updates existing records (idempotent)
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-// Will require insertPrices after implementation
-let insertPrices;
-
-let mongoServer;
-let client;
-let db;
-
-const PRICES_COLLECTION = 'unit_prices_scraper';
-
-// Mock logger for capturing log calls
-function createMockLogger() {
-  return {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn()
-  };
-}
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  ({ insertPrices } = require('../../services/scraperService'));
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean the prices collection before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-});
-
-describe('insertPrices', () => {
-  const TODAY = '2026-02-05';
-
-  // ---------------------------------------------------------------
-  // 1. Only units with non-null prices are inserted
-  // ---------------------------------------------------------------
-  describe('filtering units with null prices', () => {
-    it('should only insert records for units with non-null prices', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'A101', price: 1200 },
-        { unit_code: 'A102', price: null },
-        { unit_code: 'A103', price: 1500 },
-        { unit_code: 'A104', price: null }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(2);
-
-      const unitCodes = docs.map(d => d.unit_code).sort();
-      expect(unitCodes).toEqual(['A101', 'A103']);
-    });
-
-    it('should skip units with price of null and not create any record for them', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'B101', price: null },
-        { unit_code: 'B102', price: null }
-      ];
-
-      const result = await insertPrices(db, units, TODAY, logger);
-
-      expect(result.insertedCount).toBe(0);
-
-      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(0);
-    });
-
-    it('should handle a mix of priced and null-priced units', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'C101', price: 950 },
-        { unit_code: 'C102', price: null },
-        { unit_code: 'C103', price: 1100 },
-        { unit_code: 'C104', price: null },
-        { unit_code: 'C105', price: 1300 }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const count = await db.collection(PRICES_COLLECTION).countDocuments();
-      expect(count).toBe(3);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. bulkWrite uses updateOne with upsert for idempotency
-  // ---------------------------------------------------------------
-  describe('bulkWrite with upsert operations', () => {
-    it('should create updateOne operations for each priced unit', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'D101', price: 1200 },
-        { unit_code: 'D102', price: 1500 }
-      ];
-
-      const result = await insertPrices(db, units, TODAY, logger);
-
-      // Both units should be upserted (new inserts)
-      expect(result.insertedCount).toBeGreaterThan(0);
-
-      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(2);
-    });
-
-    it('should use upsert so re-running does not create duplicates', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'D201', price: 1200 }];
-
-      // First run
-      await insertPrices(db, units, TODAY, logger);
-
-      // Second run on same day
-      await insertPrices(db, units, TODAY, logger);
-
-      // Should still have only 1 document (not 2)
-      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(1);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. Filter uses unit_code + date_checked composite key
-  // ---------------------------------------------------------------
-  describe('composite key: unit_code + date_checked', () => {
-    it('should use unit_code and date_checked as the filter for upsert', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'E101', price: 1200 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({
-        unit_code: 'E101',
-        date_checked: TODAY
-      });
-
-      expect(doc).not.toBeNull();
-      expect(doc.unit_code).toBe('E101');
-      expect(doc.date_checked).toBe(TODAY);
-    });
-
-    it('should create separate records for the same unit on different days', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'E201', price: 1200 }];
-
-      await insertPrices(db, units, '2026-02-04', logger);
-      await insertPrices(db, units, '2026-02-05', logger);
-
-      const docs = await db.collection(PRICES_COLLECTION)
-        .find({ unit_code: 'E201' })
-        .sort({ date_checked: 1 })
-        .toArray();
-
-      expect(docs).toHaveLength(2);
-      expect(docs[0].date_checked).toBe('2026-02-04');
-      expect(docs[1].date_checked).toBe('2026-02-05');
-    });
-
-    it('should create separate records for different units on the same day', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'E301', price: 1000 },
-        { unit_code: 'E302', price: 1100 }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const docs = await db.collection(PRICES_COLLECTION)
-        .find({ date_checked: TODAY })
-        .toArray();
-
-      expect(docs).toHaveLength(2);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. Price record includes required fields
-  // ---------------------------------------------------------------
-  describe('price record fields', () => {
-    it('should include unit_code in the price record', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'F101', price: 1400 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc.unit_code).toBe('F101');
-    });
-
-    it('should include date_checked in the price record', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'F201', price: 1400 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc.date_checked).toBe(TODAY);
-    });
-
-    it('should include price as a number in the price record', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'F301', price: 1550 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc.price).toBe(1550);
-      expect(typeof doc.price).toBe('number');
-    });
-
-    it('should include last_updated as an ISO timestamp string', async () => {
-      const logger = createMockLogger();
-      const beforeTime = new Date().toISOString();
-
-      const units = [{ unit_code: 'F401', price: 1600 }];
-      await insertPrices(db, units, TODAY, logger);
-
-      const afterTime = new Date().toISOString();
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc.last_updated).toBeDefined();
-      expect(typeof doc.last_updated).toBe('string');
-      expect(doc.last_updated >= beforeTime).toBe(true);
-      expect(doc.last_updated <= afterTime).toBe(true);
-    });
-
-    it('should include data_source set to "web_scraper"', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'F501', price: 1700 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc.data_source).toBe('web_scraper');
-    });
-
-    it('should include all five required fields in every price record', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'F601', price: 1800 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const doc = await db.collection(PRICES_COLLECTION).findOne({});
-      expect(doc).toHaveProperty('unit_code');
-      expect(doc).toHaveProperty('date_checked');
-      expect(doc).toHaveProperty('price');
-      expect(doc).toHaveProperty('last_updated');
-      expect(doc).toHaveProperty('data_source');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. Empty price records logs warning and returns early
-  // ---------------------------------------------------------------
-  describe('empty price records', () => {
-    it('should log a warning when no units have prices', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'G101', price: null },
-        { unit_code: 'G102', price: null }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
-    });
-
-    it('should return insertedCount of 0 for all null-priced units', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'G201', price: null }];
-
-      const result = await insertPrices(db, units, TODAY, logger);
-
-      expect(result.insertedCount).toBe(0);
-    });
-
-    it('should log a warning when units array is empty', async () => {
-      const logger = createMockLogger();
-
-      await insertPrices(db, [], TODAY, logger);
-
-      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
-    });
-
-    it('should not perform any database writes for empty/null-priced arrays', async () => {
-      const logger = createMockLogger();
-
-      await insertPrices(db, [], TODAY, logger);
-
-      const count = await db.collection(PRICES_COLLECTION).countDocuments();
-      expect(count).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. Error handling when bulkWrite fails
-  // ---------------------------------------------------------------
-  describe('error handling', () => {
-    it('should throw error when bulkWrite fails', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'H101', price: 1000 }];
-
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
-    });
-
-    it('should log error details when bulkWrite fails', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'H201', price: 1000 }];
-
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      try {
-        await insertPrices(mockDb, units, TODAY, logger);
-      } catch (e) {
-        // Expected to throw
-      }
-
-      expect(logger.error).toHaveBeenCalledWith(
-        'Failed to insert prices',
-        expect.objectContaining({
-          errorType: 'Error',
-          errorMessage: 'Write concern timeout'
-        })
-      );
-    });
-
-    it('should re-throw the original error', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'H301', price: 1000 }];
-
-      const originalError = new TypeError('Invalid operation');
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(originalError)
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 7. Logging of inserted and updated counts
-  // ---------------------------------------------------------------
-  describe('logging of result counts', () => {
-    it('should log inserted and updated counts on success', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'I101', price: 1000 },
-        { unit_code: 'I102', price: 1100 }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Prices inserted',
-        expect.objectContaining({
-          inserted: expect.any(Number),
-          updated: expect.any(Number)
-        })
-      );
-    });
-
-    it('should log correct counts for new inserts', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'I201', price: 1000 },
-        { unit_code: 'I202', price: 1100 }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      // For new inserts: inserted=2, updated=0
-      expect(logger.info).toHaveBeenCalledWith(
-        'Prices inserted',
-        expect.objectContaining({
-          inserted: 2,
-          updated: 0
-        })
-      );
-    });
-
-    it('should log correct counts when updating existing records', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'I301', price: 1000 }];
-
-      // First insert
-      await insertPrices(db, units, TODAY, logger);
-
-      // Reset mock
-      logger.info.mockClear();
-
-      // Second run updates existing
-      const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
-      await insertPrices(db, updatedUnits, TODAY, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Prices inserted',
-        expect.objectContaining({
-          inserted: 0,
-          updated: 1
-        })
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. Re-running on same day updates existing records (idempotent)
-  // ---------------------------------------------------------------
-  describe('idempotent re-runs on same day', () => {
-    it('should update the price when re-running on the same day', async () => {
-      const logger = createMockLogger();
-
-      // First run with original price
-      await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
-
-      const doc1 = await db.collection(PRICES_COLLECTION).findOne({
-        unit_code: 'J101',
-        date_checked: TODAY
-      });
-      expect(doc1.price).toBe(1200);
-
-      // Re-run with updated price (same day)
-      await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
-
-      const doc2 = await db.collection(PRICES_COLLECTION).findOne({
-        unit_code: 'J101',
-        date_checked: TODAY
-      });
-      expect(doc2.price).toBe(1250);
-
-      // Should still have only 1 record
-      const count = await db.collection(PRICES_COLLECTION).countDocuments({
-        unit_code: 'J101',
-        date_checked: TODAY
-      });
-      expect(count).toBe(1);
-    });
-
-    it('should update last_updated when re-running on the same day', async () => {
-      const logger = createMockLogger();
-
-      // First run
-      await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
-      const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
-      const firstUpdated = doc1.last_updated;
-
-      // Small delay to ensure different timestamp
-      await new Promise(resolve => setTimeout(resolve, 10));
-
-      // Re-run on same day
-      await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
-      const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
-      const secondUpdated = doc2.last_updated;
-
-      expect(secondUpdated > firstUpdated).toBe(true);
-    });
-
-    it('should return combined insert/update count reflecting the operation', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'J301', price: 1000 },
-        { unit_code: 'J302', price: 1100 }
-      ];
-
-      // First run - inserts
-      const result1 = await insertPrices(db, units, TODAY, logger);
-      expect(result1.insertedCount).toBe(2);
-
-      // Re-run - updates
-      const result2 = await insertPrices(db, units, TODAY, logger);
-      expect(result2.insertedCount).toBe(2); // combined upserted + modified
-
-      // Still only 2 total records
-      const count = await db.collection(PRICES_COLLECTION).countDocuments();
-      expect(count).toBe(2);
-    });
-
-    it('should handle a mix of new and existing records on re-run', async () => {
-      const logger = createMockLogger();
-
-      // First run with unit A
-      await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
-
-      // Re-run with unit A (existing) and unit B (new)
-      const mixedUnits = [
-        { unit_code: 'J401', price: 1050 },
-        { unit_code: 'J402', price: 1200 }
-      ];
-      await insertPrices(db, mixedUnits, TODAY, logger);
-
-      // Should have 2 total records
-      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(2);
-
-      // Verify updated price
-      const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
-      expect(unitA.price).toBe(1050);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 9. Uses ordered: false for performance
-  // ---------------------------------------------------------------
-  describe('ordered: false for performance', () => {
-    it('should successfully process all priced units in a batch', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'K101', price: 900 },
-        { unit_code: 'K102', price: 1000 },
-        { unit_code: 'K103', price: 1100 },
-        { unit_code: 'K104', price: 1200 },
-        { unit_code: 'K105', price: 1300 }
-      ];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      const count = await db.collection(PRICES_COLLECTION).countDocuments();
-      expect(count).toBe(5);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 10. Uses correct collection
-  // ---------------------------------------------------------------
-  describe('collection usage', () => {
-    it('should write to the configured PRICES collection', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'L101', price: 1400 }];
-
-      await insertPrices(db, units, TODAY, logger);
-
-      // Verify data is in the correct collection
-      const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
-      expect(doc).not.toBeNull();
-    });
-  });
-});

__tests__/scraper/markStaleUnits.test.js

@@ -1,449 +0,0 @@
-/**
- * Tests for markStaleUnits() availability tracking
- *
- * Covers:
- * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
- * - $set updates available to false and sets marked_stale_date
- * - Units in currentCodes are NOT marked stale
- * - Already unavailable units are not modified
- * - Error handling when updateMany fails
- * - Logging of modified count
- * - Empty currentUnitCodes handles gracefully
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-// Will require markStaleUnits after implementation
-let markStaleUnits;
-
-let mongoServer;
-let client;
-let db;
-
-const UNITS_COLLECTION = 'units_scraper';
-
-// Mock logger for capturing log calls
-function createMockLogger() {
-  return {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn()
-  };
-}
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  ({ markStaleUnits } = require('../../services/scraperService'));
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean collections before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-});
-
-describe('markStaleUnits', () => {
-  // ---------------------------------------------------------------
-  // 1. updateMany is called with correct filter
-  //    (unit_code $nin currentCodes AND available: true)
-  // ---------------------------------------------------------------
-  describe('filter criteria', () => {
-    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
-      const logger = createMockLogger();
-
-      // Seed units in the database
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'A101', available: true, price: 1200 },
-        { unit_code: 'A102', available: true, price: 1300 },
-        { unit_code: 'B201', available: true, price: 1500 }
-      ]);
-
-      // Current scrape found A101 and A102, but NOT B201
-      const currentCodes = new Set(['A101', 'A102']);
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-
-      // B201 should be marked stale (it was available but not in current scrape)
-      expect(result.modifiedCount).toBe(1);
-
-      // Verify B201 was updated
-      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
-      expect(staleUnit.available).toBe(false);
-      expect(staleUnit.marked_stale_date).toBe(date);
-    });
-
-    it('should only affect units NOT in the currentUnitCodes set', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'X1', available: true },
-        { unit_code: 'X2', available: true },
-        { unit_code: 'X3', available: true },
-        { unit_code: 'X4', available: true }
-      ]);
-
-      // Current scrape found X1 and X3
-      const currentCodes = new Set(['X1', 'X3']);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      // X1 and X3 should still be available
-      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
-      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
-      expect(x1.available).toBe(true);
-      expect(x3.available).toBe(true);
-
-      // X2 and X4 should be marked stale
-      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
-      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
-      expect(x2.available).toBe(false);
-      expect(x4.available).toBe(false);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. $set updates available to false and sets marked_stale_date
-  // ---------------------------------------------------------------
-  describe('$set fields', () => {
-    it('should set available to false for stale units', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'S100', available: true, price: 1000 },
-        { unit_code: 'S200', available: true, price: 1100 }
-      ]);
-
-      const currentCodes = new Set(['S100']);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
-      expect(staleUnit.available).toBe(false);
-    });
-
-    it('should set marked_stale_date to the provided date', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'D100', available: true }
-      ]);
-
-      const currentCodes = new Set([]);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
-      expect(doc.marked_stale_date).toBe('2026-02-05');
-    });
-
-    it('should update both available and marked_stale_date in the same operation', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'C100', available: true, price: 900 }
-      ]);
-
-      const currentCodes = new Set([]);
-      const date = '2026-01-15';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
-      expect(doc.available).toBe(false);
-      expect(doc.marked_stale_date).toBe('2026-01-15');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. Units in currentCodes are NOT marked stale
-  // ---------------------------------------------------------------
-  describe('units in currentCodes not affected', () => {
-    it('should not modify units that are in the currentUnitCodes set', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'KEEP-1', available: true, price: 1200 },
-        { unit_code: 'KEEP-2', available: true, price: 1300 },
-        { unit_code: 'STALE-1', available: true, price: 1400 }
-      ]);
-
-      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      // Kept units should retain their original state
-      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
-      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
-
-      expect(keep1.available).toBe(true);
-      expect(keep1.marked_stale_date).toBeUndefined();
-      expect(keep2.available).toBe(true);
-      expect(keep2.marked_stale_date).toBeUndefined();
-    });
-
-    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'ALL-1', available: true },
-        { unit_code: 'ALL-2', available: true }
-      ]);
-
-      const currentCodes = new Set(['ALL-1', 'ALL-2']);
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-      expect(result.modifiedCount).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. Already unavailable units are not modified
-  // ---------------------------------------------------------------
-  describe('already unavailable units', () => {
-    it('should not modify units that are already unavailable (available: false)', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
-        { unit_code: 'UNAVAIL-2', available: false },
-        { unit_code: 'AVAIL-1', available: true }
-      ]);
-
-      // None of these units are in the current scrape
-      const currentCodes = new Set([]);
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-
-      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
-      expect(result.modifiedCount).toBe(1);
-
-      // Verify the already-unavailable unit's date was NOT changed
-      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
-      expect(unavail1.marked_stale_date).toBe('2026-01-01');
-    });
-
-    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'U1', available: false },
-        { unit_code: 'U2', available: false }
-      ]);
-
-      const currentCodes = new Set([]);
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-      expect(result.modifiedCount).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. Error handling when updateMany fails
-  // ---------------------------------------------------------------
-  describe('error handling', () => {
-    it('should throw error when updateMany fails', async () => {
-      const logger = createMockLogger();
-
-      const mockCollection = {
-        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      const currentCodes = new Set(['A1']);
-      const date = '2026-02-05';
-
-      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
-    });
-
-    it('should log error details when updateMany fails', async () => {
-      const logger = createMockLogger();
-
-      const mockCollection = {
-        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      const currentCodes = new Set(['A1']);
-      const date = '2026-02-05';
-
-      try {
-        await markStaleUnits(mockDb, currentCodes, date, logger);
-      } catch (e) {
-        // Expected to throw
-      }
-
-      expect(logger.error).toHaveBeenCalledWith(
-        'Failed to mark stale units',
-        expect.objectContaining({
-          errorType: 'Error',
-          errorMessage: 'Write concern timeout'
-        })
-      );
-    });
-
-    it('should re-throw the original error', async () => {
-      const logger = createMockLogger();
-
-      const originalError = new TypeError('Invalid operation');
-      const mockCollection = {
-        updateMany: jest.fn().mockRejectedValue(originalError)
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      const currentCodes = new Set(['A1']);
-      const date = '2026-02-05';
-
-      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. Logging of modified count
-  // ---------------------------------------------------------------
-  describe('logging', () => {
-    it('should log the count of stale units marked', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'LOG-1', available: true },
-        { unit_code: 'LOG-2', available: true },
-        { unit_code: 'LOG-3', available: true }
-      ]);
-
-      const currentCodes = new Set(['LOG-1']);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Stale units marked',
-        expect.objectContaining({ count: 2 })
-      );
-    });
-
-    it('should log count of 0 when no units are marked stale', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'ZERO-1', available: true }
-      ]);
-
-      const currentCodes = new Set(['ZERO-1']);
-      const date = '2026-02-05';
-
-      await markStaleUnits(db, currentCodes, date, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Stale units marked',
-        expect.objectContaining({ count: 0 })
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 7. Empty currentUnitCodes handles gracefully
-  // ---------------------------------------------------------------
-  describe('empty currentUnitCodes', () => {
-    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'EMPTY-1', available: true },
-        { unit_code: 'EMPTY-2', available: true },
-        { unit_code: 'EMPTY-3', available: true }
-      ]);
-
-      const currentCodes = new Set();
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-
-      expect(result.modifiedCount).toBe(3);
-
-      // All should be marked unavailable
-      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
-      for (const doc of docs) {
-        expect(doc.available).toBe(false);
-        expect(doc.marked_stale_date).toBe('2026-02-05');
-      }
-    });
-
-    it('should not throw when currentUnitCodes is an empty Set', async () => {
-      const logger = createMockLogger();
-
-      const currentCodes = new Set();
-      const date = '2026-02-05';
-
-      // Should not throw even with empty collection and empty set
-      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
-    });
-
-    it('should handle empty database with empty currentUnitCodes', async () => {
-      const logger = createMockLogger();
-
-      const currentCodes = new Set();
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-
-      expect(result.modifiedCount).toBe(0);
-      expect(logger.info).toHaveBeenCalledWith(
-        'Stale units marked',
-        expect.objectContaining({ count: 0 })
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. Return value
-  // ---------------------------------------------------------------
-  describe('return value', () => {
-    it('should return the MongoDB updateMany result', async () => {
-      const logger = createMockLogger();
-
-      await db.collection(UNITS_COLLECTION).insertMany([
-        { unit_code: 'RET-1', available: true },
-        { unit_code: 'RET-2', available: true }
-      ]);
-
-      const currentCodes = new Set(['RET-1']);
-      const date = '2026-02-05';
-
-      const result = await markStaleUnits(db, currentCodes, date, logger);
-
-      // updateMany result should have these standard properties
-      expect(result).toHaveProperty('matchedCount');
-      expect(result).toHaveProperty('modifiedCount');
-      expect(result).toHaveProperty('acknowledged');
-      expect(result.acknowledged).toBe(true);
-    });
-  });
-});

__tests__/scraper/parseUnits.test.js

@@ -1,517 +0,0 @@
-/**
- * Tests for parseUnits() - HTML parsing with cheerio
- *
- * Covers:
- * - Extraction of all data-spaces-* attributes
- * - Image URL extraction from nested img elements
- * - Container detection (section.spaces__tab-unit)
- * - Empty / missing container handling
- * - Deduplication by unit_code
- * - Logging behavior (info, warn, error)
- * - Missing attribute handling
- */
-
-const fs = require('fs');
-const path = require('path');
-const { parseUnits } = require('../../services/scraperService');
-
-const fixturesDir = path.join(__dirname, 'fixtures');
-
-// Helper: create a mock logger
-function createMockLogger() {
-  return {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn()
-  };
-}
-
-describe('parseUnits', () => {
-  let sampleHtml;
-  let mockLogger;
-
-  beforeAll(() => {
-    sampleHtml = fs.readFileSync(
-      path.join(fixturesDir, 'sample-listing.html'),
-      'utf-8'
-    );
-  });
-
-  beforeEach(() => {
-    mockLogger = createMockLogger();
-  });
-
-  // ---------------------------------------------------------------
-  // 1. Container detection
-  // ---------------------------------------------------------------
-  describe('container detection', () => {
-    it('should find section.spaces__tab-unit container in valid HTML', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-
-      // Should not log an error about missing container
-      expect(mockLogger.error).not.toHaveBeenCalled();
-      expect(units.length).toBeGreaterThan(0);
-    });
-
-    it('should return empty array when section container is missing', () => {
-      const html = '<html><body><div>No units here</div></body></html>';
-      const units = parseUnits(html, mockLogger);
-
-      expect(units).toEqual([]);
-      expect(mockLogger.error).toHaveBeenCalledWith(
-        expect.stringContaining('Container section.spaces__tab-unit not found'),
-        expect.any(Object)
-      );
-    });
-
-    it('should return empty array for empty HTML string', () => {
-      const units = parseUnits('', mockLogger);
-
-      expect(units).toEqual([]);
-      expect(mockLogger.error).toHaveBeenCalled();
-    });
-
-    it('should return empty array when container exists but has no articles', () => {
-      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
-      const units = parseUnits(html, mockLogger);
-
-      expect(units).toEqual([]);
-      // Container found, no error about container
-      expect(mockLogger.error).not.toHaveBeenCalled();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. Extraction of all data-spaces-* attributes
-  // ---------------------------------------------------------------
-  describe('attribute extraction', () => {
-    it('should extract all expected attributes from a fully populated article', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-      const unit1 = units.find(u => u.unit_code === 'W2707');
-
-      expect(unit1).toBeDefined();
-
-      // Core identifiers
-      expect(unit1.id).toBe('154842');
-      expect(unit1.unit_code).toBe('W2707');
-      expect(unit1.unit_id).toBe('154842');
-
-      // Physical attributes
-      expect(unit1.floor).toBe('2755');
-      expect(unit1.area).toBe('1210');
-      expect(unit1.bed_count).toBe('2');
-      expect(unit1.bath_count).toBe('2');
-
-      // Pricing
-      expect(unit1.price).toBe('5230');
-
-      // Availability
-      expect(unit1.available).toBe('true');
-      expect(unit1.unavailable).toBe('false');
-      expect(unit1.soonest).toBe('2025-10-11');
-      expect(unit1.date_available).toBe('1760140800');
-
-      // Plan information
-      expect(unit1.plan_id).toBe('10270');
-      expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
-
-      // Property information
-      expect(unit1.obj_type).toBe('unit');
-      expect(unit1.community).toBe('Country Club Towers');
-      expect(unit1.asset).toBe('420');
-
-      // URLs
-      expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
-      expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
-
-      // Specials
-      expect(unit1.specials_content).toBe('');
-    });
-
-    it('should extract correct count of units from the fixture', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-      expect(units).toHaveLength(10);
-    });
-
-    it('should return raw string values without type conversion', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-      const unit = units.find(u => u.unit_code === 'W2707');
-
-      // All values should be strings (raw extraction, no conversion)
-      expect(typeof unit.id).toBe('string');
-      expect(typeof unit.price).toBe('string');
-      expect(typeof unit.floor).toBe('string');
-      expect(typeof unit.available).toBe('string');
-      expect(typeof unit.bed_count).toBe('string');
-      expect(typeof unit.bath_count).toBe('string');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. Image URL extraction
-  // ---------------------------------------------------------------
-  describe('image URL extraction', () => {
-    it('should extract image_url from img src attribute', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
-            <img src="https://example.com/images/unit-1.jpg" />
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-
-      expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
-    });
-
-    it('should fall back to data-src when src is not present', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
-            <img data-src="https://example.com/images/unit-2.jpg" />
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-
-      expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
-    });
-
-    it('should not set image_url when no img element exists', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
-            <div>No image here</div>
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-
-      expect(units[0].image_url).toBeUndefined();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. Missing attributes
-  // ---------------------------------------------------------------
-  describe('missing attributes', () => {
-    it('should return undefined for attributes not present on the article', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article
-            data-spaces-id="1005"
-            data-spaces-unit="SPARSE-1"
-            data-spaces-unit-id="5005"
-            data-spaces-sort-price="1300"
-            data-spaces-available="true"
-            data-spaces-obj="unit"
-            data-spaces-community="Country Club Towers"
-            data-spaces-asset="100">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
-
-      expect(unit).toBeDefined();
-      expect(unit.floor).toBeUndefined();
-      expect(unit.area).toBeUndefined();
-      expect(unit.bed_count).toBeUndefined();
-      expect(unit.bath_count).toBeUndefined();
-      expect(unit.soonest).toBeUndefined();
-      expect(unit.date_available).toBeUndefined();
-      expect(unit.plan_id).toBeUndefined();
-      expect(unit.plan_name).toBeUndefined();
-      expect(unit.href).toBeUndefined();
-      expect(unit.inventory_href).toBeUndefined();
-      expect(unit.specials_content).toBeUndefined();
-    });
-
-    it('should still extract the attributes that are present on a sparse article', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article
-            data-spaces-id="1005"
-            data-spaces-unit="SPARSE-1"
-            data-spaces-unit-id="5005"
-            data-spaces-sort-price="1300"
-            data-spaces-available="true"
-            data-spaces-obj="unit"
-            data-spaces-community="Country Club Towers"
-            data-spaces-asset="100">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
-
-      expect(unit.id).toBe('1005');
-      expect(unit.unit_code).toBe('SPARSE-1');
-      expect(unit.unit_id).toBe('5005');
-      expect(unit.price).toBe('1300');
-      expect(unit.available).toBe('true');
-      expect(unit.obj_type).toBe('unit');
-      expect(unit.community).toBe('Country Club Towers');
-      expect(unit.asset).toBe('100');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. Deduplication
-  // ---------------------------------------------------------------
-  describe('deduplication', () => {
-    it('should deduplicate units with the same unit_code', () => {
-      // Build HTML with duplicate articles
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
-          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
-          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
-        </section>
-        </body></html>
-      `;
-
-      const units = parseUnits(html, mockLogger);
-      const codes = units.map(u => u.unit_code);
-
-      // Should have 2 unique unit codes
-      expect(units).toHaveLength(2);
-      expect(new Set(codes).size).toBe(codes.length);
-    });
-
-    it('should keep the first occurrence when deduplicating', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
-          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
-        </section>
-        </body></html>
-      `;
-
-      const units = parseUnits(html, mockLogger);
-
-      // First occurrence (id=1, price=1000) should be kept
-      expect(units[0].id).toBe('1');
-      expect(units[0].price).toBe('1000');
-    });
-
-    it('should log a warning when duplicates are removed', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
-          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
-          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
-        </section>
-        </body></html>
-      `;
-
-      parseUnits(html, mockLogger);
-
-      expect(mockLogger.warn).toHaveBeenCalledWith(
-        'Duplicate units removed',
-        expect.objectContaining({
-          original: 3,
-          deduplicated: 2
-        })
-      );
-    });
-
-    it('should not log a warning when there are no duplicates', () => {
-      parseUnits(sampleHtml, mockLogger);
-
-      // The fixture has no duplicates, so no warn about duplicates
-      const warnCalls = mockLogger.warn.mock.calls;
-      const duplicateWarns = warnCalls.filter(
-        call => call[0] === 'Duplicate units removed'
-      );
-      expect(duplicateWarns).toHaveLength(0);
-    });
-
-    it('should filter out articles with no unit_code during deduplication', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
-          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
-        </section>
-        </body></html>
-      `;
-
-      const units = parseUnits(html, mockLogger);
-
-      // Article without unit_code should be filtered out
-      expect(units).toHaveLength(1);
-      expect(units[0].unit_code).toBe('VALID-1');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. Logging behavior
-  // ---------------------------------------------------------------
-  describe('logging', () => {
-    it('should log the count of parsed units', () => {
-      parseUnits(sampleHtml, mockLogger);
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Units parsed',
-        expect.objectContaining({ count: 10 })
-      );
-    });
-
-    it('should log error when container is not found', () => {
-      const html = '<html><body><p>Nothing here</p></body></html>';
-      parseUnits(html, mockLogger);
-
-      expect(mockLogger.error).toHaveBeenCalledTimes(1);
-      expect(mockLogger.error).toHaveBeenCalledWith(
-        expect.stringContaining('Container section.spaces__tab-unit not found'),
-        expect.any(Object)
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
-  // ---------------------------------------------------------------
-  describe('fixture: sample-listing-empty.html', () => {
-    let emptyHtml;
-
-    beforeAll(() => {
-      emptyHtml = fs.readFileSync(
-        path.join(fixturesDir, 'sample-listing-empty.html'),
-        'utf-8'
-      );
-    });
-
-    it('should return empty array when fixture has container but no articles', () => {
-      const units = parseUnits(emptyHtml, mockLogger);
-
-      expect(units).toEqual([]);
-      // Container exists so no error about missing container
-      expect(mockLogger.error).not.toHaveBeenCalled();
-    });
-
-    it('should log zero parsed units for empty fixture', () => {
-      parseUnits(emptyHtml, mockLogger);
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Units parsed',
-        expect.objectContaining({ count: 0 })
-      );
-    });
-  });
-
-  describe('fixture: sample-listing-call.html', () => {
-    let callHtml;
-
-    beforeAll(() => {
-      callHtml = fs.readFileSync(
-        path.join(fixturesDir, 'sample-listing-call.html'),
-        'utf-8'
-      );
-    });
-
-    it('should extract both units from the call-for-pricing fixture', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units).toHaveLength(2);
-      expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
-    });
-
-    it('should preserve "Call for pricing" as raw string in the price field', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units[0].price).toBe('Call for pricing');
-      expect(units[1].price).toBe('Call for pricing');
-    });
-
-    it('should extract all attributes correctly from call-for-pricing units', () => {
-      const units = parseUnits(callHtml, mockLogger);
-      const unit = units.find(u => u.unit_code === 'P-101');
-
-      expect(unit.id).toBe('9001');
-      expect(unit.unit_id).toBe('9001');
-      expect(unit.floor).toBe('1');
-      expect(unit.area).toBe('750');
-      expect(unit.bed_count).toBe('1');
-      expect(unit.bath_count).toBe('1');
-      expect(unit.available).toBe('true');
-      expect(unit.unavailable).toBe('false');
-      expect(unit.soonest).toBe('Now');
-      expect(unit.date_available).toBe('1706745600');
-      expect(unit.plan_id).toBe('500');
-      expect(unit.plan_name).toBe('Penthouse A');
-      expect(unit.community).toBe('Country Club Towers');
-      expect(unit.asset).toBe('420');
-      expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
-      expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
-      expect(unit.specials_content).toBe('');
-    });
-
-    it('should not have image_url when fixture units lack img elements', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units[0].image_url).toBeUndefined();
-      expect(units[1].image_url).toBeUndefined();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. Multiple units with varied data
-  // ---------------------------------------------------------------
-  describe('varied unit data', () => {
-    it('should handle unavailable units correctly', () => {
-      const html = `
-        <html><body>
-        <section class="spaces__tab-unit">
-          <article
-            data-spaces-id="4001"
-            data-spaces-unit="UNAVAIL-1"
-            data-spaces-sort-price="2000"
-            data-spaces-available="false"
-            data-spaces-unavailable="true"
-            data-spaces-soonest=""
-            data-spaces-sort-date="">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
-
-      expect(unit.available).toBe('false');
-      expect(unit.unavailable).toBe('true');
-      expect(unit.soonest).toBe('');
-      expect(unit.date_available).toBe('');
-    });
-
-    it('should handle half bath counts', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-      // E3205 is the penthouse with 2.5 baths
-      const unit = units.find(u => u.unit_code === 'E3205');
-
-      expect(unit.bath_count).toBe('2.5');
-    });
-
-    it('should handle studio units (bed_count = 0)', () => {
-      const units = parseUnits(sampleHtml, mockLogger);
-      // W2603 is the studio with bed_count 0
-      const unit = units.find(u => u.unit_code === 'W2603');
-
-      expect(unit.bed_count).toBe('0');
-    });
-  });
-});

__tests__/scraper/recordScraperRun.test.js

@@ -1,307 +0,0 @@
-/**
- * Tests for recordScraperRun() history persistence
- *
- * Covers:
- * - insertOne is called with runData plus recordedAt timestamp
- * - All runData fields are preserved (jobId, trigger, status, duration, etc.)
- * - Error is caught and logged but NOT thrown (graceful failure)
- * - Returns null on error instead of crashing
- * - Returns insert result on success
- * - recordedAt is a valid Date object
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-// Will require after implementation
-let recordScraperRun;
-
-let mongoServer;
-let client;
-let db;
-let logger;
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  ({ recordScraperRun } = require('../../services/scraperService'));
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean all collections before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-
-  // Create fresh mock logger for each test
-  logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
-});
-
-describe('recordScraperRun', () => {
-  const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
-
-  // Sample runData matching the structure from runScrape()'s finally block
-  function createSampleRunData(overrides = {}) {
-    return {
-      jobId: 'test-job-001',
-      trigger: 'manual',
-      dryRun: false,
-      status: 'success',
-      startedAt: '2026-02-06T06:00:00.000Z',
-      completedAt: '2026-02-06T06:00:12.345Z',
-      duration: 12345,
-      unitsProcessed: 50,
-      pricesInserted: 48,
-      newUnitsCount: 3,
-      rentedUnitsCount: 1,
-      staleUnitsCount: 2,
-      errors: [],
-      ...overrides
-    };
-  }
-
-  // ---------------------------------------------------------------
-  // 1. insertOne is called with runData plus recordedAt timestamp
-  // ---------------------------------------------------------------
-  describe('insertOne with runData and recordedAt', () => {
-    it('should insert a document into the scraper_runs collection', async () => {
-      const runData = createSampleRunData();
-
-      await recordScraperRun(db, runData, logger);
-
-      const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(1);
-    });
-
-    it('should include a recordedAt field in the inserted document', async () => {
-      const runData = createSampleRunData();
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.recordedAt).toBeDefined();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. All runData fields are preserved
-  // ---------------------------------------------------------------
-  describe('preserving all runData fields', () => {
-    it('should preserve jobId, trigger, status, and duration', async () => {
-      const runData = createSampleRunData({
-        jobId: 'preserve-test-001',
-        trigger: 'scheduled',
-        status: 'failed',
-        duration: 9999
-      });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.jobId).toBe('preserve-test-001');
-      expect(doc.trigger).toBe('scheduled');
-      expect(doc.status).toBe('failed');
-      expect(doc.duration).toBe(9999);
-    });
-
-    it('should preserve unit processing metrics', async () => {
-      const runData = createSampleRunData({
-        unitsProcessed: 42,
-        pricesInserted: 40,
-        newUnitsCount: 5,
-        rentedUnitsCount: 2,
-        staleUnitsCount: 3
-      });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.unitsProcessed).toBe(42);
-      expect(doc.pricesInserted).toBe(40);
-      expect(doc.newUnitsCount).toBe(5);
-      expect(doc.rentedUnitsCount).toBe(2);
-      expect(doc.staleUnitsCount).toBe(3);
-    });
-
-    it('should preserve timing fields (startedAt, completedAt)', async () => {
-      const runData = createSampleRunData({
-        startedAt: '2026-02-06T10:00:00.000Z',
-        completedAt: '2026-02-06T10:00:15.500Z'
-      });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
-      expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
-    });
-
-    it('should preserve dryRun flag', async () => {
-      const runData = createSampleRunData({ dryRun: true });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.dryRun).toBe(true);
-    });
-
-    it('should preserve errors array when it has entries', async () => {
-      const runData = createSampleRunData({
-        status: 'failed',
-        errors: ['Connection timeout', 'Retry exhausted']
-      });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
-    });
-
-    it('should preserve empty errors array for successful runs', async () => {
-      const runData = createSampleRunData({ errors: [] });
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.errors).toEqual([]);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. Error is caught and logged but NOT thrown (graceful failure)
-  // ---------------------------------------------------------------
-  describe('graceful error handling', () => {
-    it('should NOT throw when insertOne fails', async () => {
-      const runData = createSampleRunData();
-
-      // Create a mock db that throws on insertOne
-      const mockCollection = {
-        insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      // This should NOT throw
-      await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
-    });
-
-    it('should log error message when insert fails', async () => {
-      const runData = createSampleRunData();
-
-      const mockCollection = {
-        insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await recordScraperRun(mockDb, runData, logger);
-
-      expect(logger.error).toHaveBeenCalledWith(
-        'Failed to record scraper run',
-        { errorMessage: 'Disk full' }
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. Returns null on error instead of crashing
-  // ---------------------------------------------------------------
-  describe('return value on error', () => {
-    it('should return null when insertOne throws', async () => {
-      const runData = createSampleRunData();
-
-      const mockCollection = {
-        insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      const result = await recordScraperRun(mockDb, runData, logger);
-      expect(result).toBeNull();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. Returns insert result on success
-  // ---------------------------------------------------------------
-  describe('return value on success', () => {
-    it('should return the insertOne result object', async () => {
-      const runData = createSampleRunData();
-
-      const result = await recordScraperRun(db, runData, logger);
-
-      expect(result).toBeDefined();
-      expect(result).not.toBeNull();
-    });
-
-    it('should return result with acknowledged property', async () => {
-      const runData = createSampleRunData();
-
-      const result = await recordScraperRun(db, runData, logger);
-
-      expect(result.acknowledged).toBe(true);
-    });
-
-    it('should return result with insertedId', async () => {
-      const runData = createSampleRunData();
-
-      const result = await recordScraperRun(db, runData, logger);
-
-      expect(result.insertedId).toBeDefined();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. recordedAt is a valid Date object
-  // ---------------------------------------------------------------
-  describe('recordedAt field', () => {
-    it('should set recordedAt as a Date instance', async () => {
-      const runData = createSampleRunData();
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.recordedAt).toBeInstanceOf(Date);
-    });
-
-    it('should set recordedAt close to current time', async () => {
-      const beforeTime = new Date();
-
-      const runData = createSampleRunData();
-      await recordScraperRun(db, runData, logger);
-
-      const afterTime = new Date();
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-      expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
-      expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
-    });
-
-    it('should not overwrite any existing runData fields with recordedAt', async () => {
-      const runData = createSampleRunData();
-
-      await recordScraperRun(db, runData, logger);
-
-      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
-
-      // Verify the original fields still exist alongside recordedAt
-      expect(doc.jobId).toBe(runData.jobId);
-      expect(doc.status).toBe(runData.status);
-      expect(doc.recordedAt).toBeInstanceOf(Date);
-    });
-  });
-});

__tests__/scraper/scraperConfig.test.js

@@ -1,192 +0,0 @@
-/**
- * Tests for config/scraper.js
- * Tests scraper configuration constants and environment variable overrides
- */
-
-describe('config/scraper', () => {
-  // Store original env values to restore after tests
-  const originalEnv = { ...process.env };
-
-  beforeEach(() => {
-    // Clear all scraper-related env vars before each test
-    delete process.env.SCRAPER_SCHEDULE;
-    delete process.env.SCRAPER_TIMEZONE;
-    delete process.env.SCRAPER_ENABLED;
-    delete process.env.SCRAPER_TIMEOUT;
-    delete process.env.SCRAPER_UNITS_COLLECTION;
-    delete process.env.SCRAPER_PRICES_COLLECTION;
-    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
-    delete process.env.SCRAPER_RUNS_COLLECTION;
-
-    // Clear the require cache to reload config with fresh env
-    jest.resetModules();
-  });
-
-  afterAll(() => {
-    // Restore original environment
-    process.env = originalEnv;
-  });
-
-  describe('TARGET_URL', () => {
-    test('should be set to the correct apartment listing URL', () => {
-      const config = require('../../config/scraper');
-      expect(config.TARGET_URL).toBe(
-        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
-      );
-    });
-  });
-
-  describe('SCRAPER_SCHEDULE', () => {
-    test('should default to "0 6 * * *" when env var not set', () => {
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
-    });
-
-    test('should use env var override when set', () => {
-      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
-    });
-  });
-
-  describe('SCRAPER_TIMEZONE', () => {
-    test('should default to "America/Denver" when env var not set', () => {
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
-    });
-
-    test('should use env var override when set', () => {
-      process.env.SCRAPER_TIMEZONE = 'America/New_York';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEZONE).toBe('America/New_York');
-    });
-  });
-
-  describe('SCRAPER_ENABLED', () => {
-    test('should default to true when env var not set', () => {
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_ENABLED).toBe(true);
-    });
-
-    test('should be false when env var is set to "false"', () => {
-      process.env.SCRAPER_ENABLED = 'false';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_ENABLED).toBe(false);
-    });
-
-    test('should be true when env var is set to any other value', () => {
-      process.env.SCRAPER_ENABLED = 'true';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_ENABLED).toBe(true);
-    });
-
-    test('should be true when env var is empty string', () => {
-      process.env.SCRAPER_ENABLED = '';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_ENABLED).toBe(true);
-    });
-  });
-
-  describe('SCRAPER_TIMEOUT', () => {
-    test('should default to 30000 when env var not set', () => {
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEOUT).toBe(30000);
-    });
-
-    test('should use env var override when set', () => {
-      process.env.SCRAPER_TIMEOUT = '60000';
-      const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEOUT).toBe(60000);
-    });
-
-    test('should parse string to integer', () => {
-      process.env.SCRAPER_TIMEOUT = '45000';
-      const config = require('../../config/scraper');
-      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
-      expect(config.SCRAPER_TIMEOUT).toBe(45000);
-    });
-  });
-
-  describe('USER_AGENT', () => {
-    test('should be a valid User-Agent string', () => {
-      const config = require('../../config/scraper');
-      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
-    });
-  });
-
-  describe('RETRY_CONFIG', () => {
-    test('should have maxRetries property', () => {
-      const config = require('../../config/scraper');
-      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
-      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
-    });
-
-    test('should have baseDelay property', () => {
-      const config = require('../../config/scraper');
-      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
-      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
-    });
-
-    test('should have timeout property', () => {
-      const config = require('../../config/scraper');
-      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
-      expect(config.RETRY_CONFIG.timeout).toBe(30000);
-    });
-
-    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
-      process.env.SCRAPER_TIMEOUT = '45000';
-      const config = require('../../config/scraper');
-      expect(config.RETRY_CONFIG.timeout).toBe(45000);
-    });
-  });
-
-  describe('COLLECTIONS', () => {
-    describe('default values', () => {
-      test('UNITS should default to "units_scraper"', () => {
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
-      });
-
-      test('PRICES should default to "unit_prices_scraper"', () => {
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
-      });
-
-      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
-      });
-
-      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
-      });
-    });
-
-    describe('env var overrides', () => {
-      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
-        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
-      });
-
-      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
-        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
-      });
-
-      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
-        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
-      });
-
-      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
-        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
-        const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
-      });
-    });
-  });
-});

__tests__/scraper/scraperLogger.test.js

@@ -1,471 +0,0 @@
-/**
- * Tests for scraperLogger.js
- * Following TDD - these tests are written BEFORE implementation
- */
-
-const { createLogger, LEVELS } = require('../../services/scraperLogger');
-
-describe('scraperLogger', () => {
-  let consoleSpy;
-  let logOutput;
-
-  beforeEach(() => {
-    logOutput = [];
-    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
-      logOutput.push(output);
-    });
-  });
-
-  afterEach(() => {
-    consoleSpy.mockRestore();
-  });
-
-  describe('createLogger', () => {
-    test('should return an object with info, warn, error methods', () => {
-      const logger = createLogger('test-job-id');
-
-      expect(typeof logger).toBe('object');
-      expect(typeof logger.info).toBe('function');
-      expect(typeof logger.warn).toBe('function');
-      expect(typeof logger.error).toBe('function');
-    });
-
-    test('should include jobId in all log entries', () => {
-      const jobId = 'unique-job-123';
-      const logger = createLogger(jobId);
-
-      logger.info('Test message');
-
-      expect(logOutput.length).toBe(1);
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.jobId).toBe(jobId);
-    });
-  });
-
-  describe('log entry format', () => {
-    test('should include timestamp in ISO format', () => {
-      const logger = createLogger('test-job');
-      const beforeTime = new Date().toISOString();
-
-      logger.info('Test message');
-
-      const afterTime = new Date().toISOString();
-      const parsed = JSON.parse(logOutput[0]);
-
-      expect(parsed.timestamp).toBeDefined();
-      // Verify timestamp is in ISO format
-      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
-      // Verify timestamp is within test window
-      expect(parsed.timestamp >= beforeTime).toBe(true);
-      expect(parsed.timestamp <= afterTime).toBe(true);
-    });
-
-    test('should include log level', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Info message');
-      logger.warn('Warn message');
-      logger.error('Error message');
-
-      expect(logOutput.length).toBe(3);
-
-      const infoEntry = JSON.parse(logOutput[0]);
-      const warnEntry = JSON.parse(logOutput[1]);
-      const errorEntry = JSON.parse(logOutput[2]);
-
-      expect(infoEntry.level).toBe('info');
-      expect(warnEntry.level).toBe('warn');
-      expect(errorEntry.level).toBe('error');
-    });
-
-    test('should include message', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Test message content');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.message).toBe('Test message content');
-    });
-
-    test('should include context when provided', () => {
-      const logger = createLogger('test-job');
-      const context = { key: 'value', count: 42 };
-
-      logger.info('Test message', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context).toEqual(context);
-    });
-
-    test('should output valid JSON', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Test message', { data: 'test' });
-
-      expect(() => JSON.parse(logOutput[0])).not.toThrow();
-    });
-  });
-
-  describe('log levels', () => {
-    test('info() should output with level "info"', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Info message');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.level).toBe('info');
-    });
-
-    test('warn() should output with level "warn"', () => {
-      const logger = createLogger('test-job');
-
-      logger.warn('Warning message');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.level).toBe('warn');
-    });
-
-    test('error() should output with level "error"', () => {
-      const logger = createLogger('test-job');
-
-      logger.error('Error message');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.level).toBe('error');
-    });
-  });
-
-  describe('LEVELS constant', () => {
-    test('should export LEVELS constant with info, warn, error', () => {
-      expect(LEVELS).toBeDefined();
-      expect(LEVELS.info).toBe('info');
-      expect(LEVELS.warn).toBe('warn');
-      expect(LEVELS.error).toBe('error');
-    });
-  });
-
-  describe('message truncation', () => {
-    test('should truncate messages longer than 1000 characters', () => {
-      const logger = createLogger('test-job');
-      const longMessage = 'x'.repeat(1500);
-
-      logger.info(longMessage);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
-      expect(parsed.message).toContain('... [truncated]');
-    });
-
-    test('should not truncate messages of 1000 characters or less', () => {
-      const logger = createLogger('test-job');
-      const shortMessage = 'x'.repeat(1000);
-
-      logger.info(shortMessage);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.message).toBe(shortMessage);
-      expect(parsed.message).not.toContain('... [truncated]');
-    });
-
-    test('should handle empty message', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.message).toBe('');
-    });
-
-    test('should handle null or undefined message', () => {
-      const logger = createLogger('test-job');
-
-      logger.info(null);
-      logger.info(undefined);
-
-      expect(logOutput.length).toBe(2);
-      // Should not throw and should handle gracefully
-    });
-  });
-
-  describe('MongoDB connection string redaction', () => {
-    test('should redact username and password from mongodb:// URIs', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
-      };
-
-      logger.info('Database connection', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
-      expect(parsed.context.uri).not.toContain('admin');
-      expect(parsed.context.uri).not.toContain('secretpassword123');
-    });
-
-    test('should redact username and password from mongodb+srv:// URIs', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
-      };
-
-      logger.info('Database connection', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
-      expect(parsed.context.uri).not.toContain('myuser');
-      expect(parsed.context.uri).not.toContain('mypass');
-    });
-
-    test('should handle connection string without credentials', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        uri: 'mongodb://localhost:27017/mydb'
-      };
-
-      logger.info('Database connection', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
-    });
-  });
-
-  describe('sensitive key redaction', () => {
-    test('should redact keys containing "password"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        password: 'secret123',
-        userPassword: 'anotherSecret',
-        password_hash: 'hashed'
-      };
-
-      logger.info('User data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.password).toBe('[REDACTED]');
-      expect(parsed.context.userPassword).toBe('[REDACTED]');
-      expect(parsed.context.password_hash).toBe('[REDACTED]');
-    });
-
-    test('should redact keys containing "secret"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        secret: 'mysecret',
-        clientSecret: 'secret123',
-        SECRET_KEY: 'key'
-      };
-
-      logger.info('Config data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.secret).toBe('[REDACTED]');
-      expect(parsed.context.clientSecret).toBe('[REDACTED]');
-      // Case-insensitive check
-    });
-
-    test('should redact keys containing "token"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        token: 'abc123',
-        accessToken: 'token456',
-        refresh_token: 'refresh789'
-      };
-
-      logger.info('Auth data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.token).toBe('[REDACTED]');
-      expect(parsed.context.accessToken).toBe('[REDACTED]');
-      expect(parsed.context.refresh_token).toBe('[REDACTED]');
-    });
-
-    test('should redact keys containing "apikey"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        apikey: 'key123',
-        apiKey: 'key456',
-        api_key: 'key789' // Note: underscores may not match 'apikey'
-      };
-
-      logger.info('API data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.apikey).toBe('[REDACTED]');
-      expect(parsed.context.apiKey).toBe('[REDACTED]');
-    });
-
-    test('should redact keys containing "authorization"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        authorization: 'Bearer token123',
-        Authorization: 'Basic base64string'
-      };
-
-      logger.info('Header data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.authorization).toBe('[REDACTED]');
-    });
-
-    test('should not redact non-sensitive keys', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        username: 'john',
-        email: 'john@example.com',
-        count: 42
-      };
-
-      logger.info('User info', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.username).toBe('john');
-      expect(parsed.context.email).toBe('john@example.com');
-      expect(parsed.context.count).toBe(42);
-    });
-  });
-
-  describe('circular reference handling', () => {
-    test('should handle circular references in context', () => {
-      const logger = createLogger('test-job');
-      const context = { name: 'test' };
-      context.self = context; // Create circular reference
-
-      // Should not throw
-      expect(() => {
-        logger.info('Circular test', context);
-      }).not.toThrow();
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.name).toBe('test');
-      expect(parsed.context.self).toBe('[Circular]');
-    });
-
-    test('should handle deeply nested circular references', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        level1: {
-          level2: {
-            level3: {}
-          }
-        }
-      };
-      context.level1.level2.level3.back = context.level1;
-
-      expect(() => {
-        logger.info('Deep circular test', context);
-      }).not.toThrow();
-
-      // Should be valid JSON output
-      expect(() => JSON.parse(logOutput[0])).not.toThrow();
-    });
-  });
-
-  describe('Buffer handling', () => {
-    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        data: Buffer.from('Hello World')
-      };
-
-      logger.info('Buffer data', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
-    });
-
-    test('should handle empty Buffer', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        data: Buffer.alloc(0)
-      };
-
-      logger.info('Empty buffer', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
-    });
-
-    test('should handle large Buffer', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        data: Buffer.alloc(1024 * 1024) // 1MB buffer
-      };
-
-      logger.info('Large buffer', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
-    });
-  });
-
-  describe('edge cases', () => {
-    test('should handle empty context object', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Test message', {});
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context).toEqual({});
-    });
-
-    test('should handle context not provided', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Test message');
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context).toEqual({});
-    });
-
-    test('should handle null context', () => {
-      const logger = createLogger('test-job');
-
-      logger.info('Test message', null);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context).toEqual({});
-    });
-
-    test('should handle undefined values in context', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        defined: 'value',
-        undefinedValue: undefined
-      };
-
-      logger.info('Test message', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.defined).toBe('value');
-      // undefined is typically converted to null in JSON
-      expect(parsed.context.undefinedValue).toBeNull();
-    });
-
-    test('should handle nested objects in context', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        outer: {
-          inner: {
-            value: 'deep'
-          }
-        }
-      };
-
-      logger.info('Nested test', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.outer.inner.value).toBe('deep');
-    });
-
-    test('should handle arrays in context', () => {
-      const logger = createLogger('test-job');
-      const context = {
-        items: [1, 2, 3, 'four']
-      };
-
-      logger.info('Array test', context);
-
-      const parsed = JSON.parse(logOutput[0]);
-      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
-    });
-  });
-});

__tests__/scraper/scraperService.test.js

@@ -1,739 +0,0 @@
-/**
- * Tests for runScrape() orchestration function
- *
- * Covers:
- * - Full workflow completes with mocked dependencies
- * - Returns correct result structure with jobId, status, metrics
- * - Handles dryRun option (skip DB writes)
- * - Handles htmlContent option (use provided HTML)
- * - Records scraper run to history on success
- * - Records scraper run to history on failure
- * - Catches and logs errors from fetchPage
- * - Catches and logs errors from database operations
- * - Calculates newUnitsCount and rentedUnitsCount correctly
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-let mongoServer;
-let client;
-let db;
-
-// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
-let runScrape;
-let recordScraperRun;
-let createScraperIndexes;
-
-// Store original module references so we can mock individual functions
-let scraperService;
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  scraperService = require('../../services/scraperService');
-  ({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean all relevant collections before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-  // Reset all mocks
-  jest.restoreAllMocks();
-});
-
-// ============================================================
-// Helper: Create sample HTML with units
-// ============================================================
-function createSampleHtml(unitCodes) {
-  const articles = unitCodes.map(code => `
-    <article
-      data-spaces-id="100"
-      data-spaces-unit="${code}"
-      data-spaces-unit-id="200"
-      data-spaces-unit-floor="5"
-      data-spaces-sort-area="750"
-      data-spaces-sort-bed="1"
-      data-spaces-sort-bath="1"
-      data-spaces-sort-price="1500"
-      data-spaces-available="true"
-      data-spaces-unavailable="false"
-      data-spaces-soonest="Now"
-      data-spaces-sort-date="1700000000"
-      data-spaces-plan-id="10"
-      data-spaces-sort-plan-name="Studio"
-      data-spaces-obj="unit"
-      data-spaces-community="TestCommunity"
-      data-spaces-asset="1"
-      data-spaces-href="/unit/${code}"
-      data-spaces-inventory-href="/inventory/${code}"
-    >
-      <img src="http://example.com/${code}.jpg" />
-    </article>
-  `).join('\n');
-
-  return `
-    <html><body>
-      <section class="spaces__tab-unit">
-        ${articles}
-      </section>
-    </body></html>
-  `;
-}
-
-// ============================================================
-// Test: recordScraperRun
-// ============================================================
-describe('recordScraperRun', () => {
-  it('should insert a run record into scraper_runs collection', async () => {
-    const runData = {
-      jobId: 'test-job-001',
-      trigger: 'manual',
-      status: 'success',
-      startedAt: new Date().toISOString(),
-      completedAt: new Date().toISOString(),
-      duration: 1234,
-      unitsProcessed: 10,
-      pricesInserted: 8,
-      errors: []
-    };
-
-    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
-    const result = await recordScraperRun(db, runData, mockLogger);
-
-    expect(result).toBeTruthy();
-    expect(result.insertedId).toBeTruthy();
-
-    // Verify it was inserted
-    const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
-    expect(saved).toBeTruthy();
-    expect(saved.status).toBe('success');
-    expect(saved.recordedAt).toBeInstanceOf(Date);
-  });
-
-  it('should not throw when insert fails', async () => {
-    // Pass null db to cause an error
-    const runData = { jobId: 'test-fail', status: 'success' };
-
-    // Should not throw
-    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
-    const result = await recordScraperRun(null, runData, mockLogger);
-    expect(result).toBeNull();
-    expect(mockLogger.error).toHaveBeenCalled();
-  });
-});
-
-// ============================================================
-// Test: runScrape - Full workflow
-// ============================================================
-describe('runScrape', () => {
-  it('should complete full workflow with mocked dependencies', async () => {
-    const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
-
-    const result = await runScrape(db, {
-      trigger: 'manual',
-      jobId: 'test-full-workflow',
-      htmlContent: html
-    });
-
-    expect(result).toBeTruthy();
-    expect(result.jobId).toBe('test-full-workflow');
-    expect(result.status).toBe('success');
-    expect(result.trigger).toBe('manual');
-    expect(result.unitsProcessed).toBe(2);
-    expect(result.errors).toEqual([]);
-  });
-
-  it('should return correct result structure with jobId, status, metrics', async () => {
-    const html = createSampleHtml(['UNIT-X']);
-
-    const result = await runScrape(db, {
-      jobId: 'test-structure',
-      htmlContent: html
-    });
-
-    // Verify all expected fields exist
-    expect(result).toHaveProperty('jobId', 'test-structure');
-    expect(result).toHaveProperty('trigger', 'manual');
-    expect(result).toHaveProperty('dryRun', false);
-    expect(result).toHaveProperty('status', 'success');
-    expect(result).toHaveProperty('startedAt');
-    expect(result).toHaveProperty('completedAt');
-    expect(result).toHaveProperty('duration');
-    expect(result).toHaveProperty('unitsProcessed');
-    expect(result).toHaveProperty('pricesInserted');
-    expect(result).toHaveProperty('newUnitsCount');
-    expect(result).toHaveProperty('rentedUnitsCount');
-    expect(result).toHaveProperty('staleUnitsCount');
-    expect(result).toHaveProperty('errors');
-
-    // Verify types
-    expect(typeof result.duration).toBe('number');
-    expect(result.duration).toBeGreaterThanOrEqual(0);
-    expect(typeof result.startedAt).toBe('string');
-    expect(typeof result.completedAt).toBe('string');
-    expect(Array.isArray(result.errors)).toBe(true);
-  });
-
-  it('should generate a jobId when not provided', async () => {
-    const html = createSampleHtml(['UNIT-GEN']);
-
-    const result = await runScrape(db, {
-      htmlContent: html
-    });
-
-    expect(result.jobId).toBeTruthy();
-    expect(typeof result.jobId).toBe('string');
-    // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-    expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
-  });
-
-  // ============================================================
-  // Test: dryRun option
-  // ============================================================
-  it('should skip DB writes when dryRun is true', async () => {
-    const html = createSampleHtml(['DRY-A', 'DRY-B']);
-
-    const result = await runScrape(db, {
-      jobId: 'test-dry-run',
-      htmlContent: html,
-      dryRun: true
-    });
-
-    expect(result.status).toBe('success');
-    expect(result.dryRun).toBe(true);
-    expect(result.unitsProcessed).toBe(2);
-    expect(result.pricesInserted).toBe(0);
-    expect(result.staleUnitsCount).toBe(0);
-
-    // Verify no units were written to the units collection
-    const unitsCount = await db.collection('units_scraper').countDocuments();
-    expect(unitsCount).toBe(0);
-
-    // Verify no prices were written
-    const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
-    expect(pricesCount).toBe(0);
-  });
-
-  // ============================================================
-  // Test: htmlContent option
-  // ============================================================
-  it('should use provided HTML instead of fetching when htmlContent is given', async () => {
-    const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
-
-    // Mock axios.get to track if HTTP fetch is attempted
-    const axios = require('axios');
-    const axiosSpy = jest.spyOn(axios, 'get');
-
-    const result = await runScrape(db, {
-      jobId: 'test-html-content',
-      htmlContent: html
-    });
-
-    expect(result.status).toBe('success');
-    expect(result.unitsProcessed).toBe(3);
-    // axios.get should NOT have been called since we provided htmlContent
-    expect(axiosSpy).not.toHaveBeenCalled();
-  });
-
-  // ============================================================
-  // Test: Records scraper run on success
-  // ============================================================
-  it('should record scraper run to history on success', async () => {
-    const html = createSampleHtml(['REC-A']);
-
-    await runScrape(db, {
-      jobId: 'test-record-success',
-      htmlContent: html
-    });
-
-    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
-    expect(runRecord).toBeTruthy();
-    expect(runRecord.status).toBe('success');
-    expect(runRecord.recordedAt).toBeInstanceOf(Date);
-    expect(runRecord.unitsProcessed).toBe(1);
-  });
-
-  // ============================================================
-  // Test: Records scraper run on failure
-  // ============================================================
-  it('should record scraper run to history on failure', async () => {
-    const html = createSampleHtml(['FAIL-A']);
-
-    // Create a db proxy that throws on bulkWrite (used by upsertUnits)
-    // but allows other operations (like scraper_runs insertOne) to pass through
-    const faultyDb = {
-      collection: (name) => {
-        const realCollection = db.collection(name);
-        if (name === 'units_scraper') {
-          return new Proxy(realCollection, {
-            get(target, prop) {
-              if (prop === 'bulkWrite') {
-                return async () => {
-                  throw new Error('Database connection lost');
-                };
-              }
-              const value = target[prop];
-              if (typeof value === 'function') {
-                return value.bind(target);
-              }
-              return value;
-            }
-          });
-        }
-        return realCollection;
-      }
-    };
-
-    const result = await runScrape(faultyDb, {
-      jobId: 'test-record-failure',
-      htmlContent: html
-    });
-
-    expect(result.status).toBe('failed');
-    expect(result.errors).toContain('Database connection lost');
-
-    // The run should still be recorded (via the real db passed through for scraper_runs)
-    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
-    expect(runRecord).toBeTruthy();
-    expect(runRecord.status).toBe('failed');
-    expect(runRecord.errors).toContain('Database connection lost');
-  });
-
-  // ============================================================
-  // Test: Catches errors from fetchPage
-  // ============================================================
-  it('should catch and handle errors from fetchPage', async () => {
-    // Mock axios.get to throw a network error (fetchPage uses axios internally)
-    const axios = require('axios');
-    jest.spyOn(axios, 'get').mockRejectedValue(
-      new Error('Network timeout')
-    );
-
-    const result = await runScrape(db, {
-      jobId: 'test-fetch-error'
-      // No htmlContent, so it will call fetchPage which uses axios
-    });
-
-    expect(result.status).toBe('failed');
-    expect(result.errors).toContain('Network timeout');
-    expect(result.unitsProcessed).toBe(0);
-  });
-
-  // ============================================================
-  // Test: Catches errors from database operations
-  // ============================================================
-  it('should catch and handle errors from database operations', async () => {
-    const html = createSampleHtml(['DB-ERR']);
-
-    // Create a db proxy that throws on the prices collection bulkWrite
-    // Use Proxy to properly delegate all methods to the real collection
-    const faultyDb = {
-      collection: (name) => {
-        const realCollection = db.collection(name);
-        if (name === 'unit_prices_scraper') {
-          return new Proxy(realCollection, {
-            get(target, prop) {
-              if (prop === 'bulkWrite') {
-                return async () => {
-                  throw new Error('Write concern timeout');
-                };
-              }
-              const value = target[prop];
-              if (typeof value === 'function') {
-                return value.bind(target);
-              }
-              return value;
-            }
-          });
-        }
-        return realCollection;
-      }
-    };
-
-    const result = await runScrape(faultyDb, {
-      jobId: 'test-db-error',
-      htmlContent: html
-    });
-
-    expect(result.status).toBe('failed');
-    expect(result.errors).toContain('Write concern timeout');
-
-    // Run should still be recorded despite error
-    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
-    expect(runRecord).toBeTruthy();
-    expect(runRecord.status).toBe('failed');
-  });
-
-  // ============================================================
-  // Test: Calculates newUnitsCount and rentedUnitsCount correctly
-  // ============================================================
-  it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
-    // First, simulate yesterday's data by inserting price records for yesterday
-    const today = new Date();
-    const yesterday = new Date(today);
-    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
-    const yesterdayStr = yesterday.toISOString().split('T')[0];
-
-    // Yesterday had units: OLD-A, OLD-B, OLD-C
-    await db.collection('unit_prices_scraper').insertMany([
-      { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
-      { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
-      { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
-    ]);
-
-    // Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
-    const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
-
-    const result = await runScrape(db, {
-      jobId: 'test-calc-changes',
-      htmlContent: html
-    });
-
-    expect(result.status).toBe('success');
-    // NEW-D is new (not in yesterday's data)
-    expect(result.newUnitsCount).toBe(1);
-    // OLD-C was in yesterday's data but not in today's scrape
-    expect(result.rentedUnitsCount).toBe(1);
-    expect(result.unitsProcessed).toBe(3);
-  });
-
-  // ============================================================
-  // Test: Default trigger is 'manual'
-  // ============================================================
-  it('should default trigger to manual', async () => {
-    const html = createSampleHtml(['DEF-A']);
-
-    const result = await runScrape(db, {
-      jobId: 'test-default-trigger',
-      htmlContent: html
-    });
-
-    expect(result.trigger).toBe('manual');
-  });
-
-  // ============================================================
-  // Test: Supports scheduled trigger
-  // ============================================================
-  it('should support scheduled trigger', async () => {
-    const html = createSampleHtml(['SCHED-A']);
-
-    const result = await runScrape(db, {
-      jobId: 'test-scheduled-trigger',
-      trigger: 'scheduled',
-      htmlContent: html
-    });
-
-    expect(result.trigger).toBe('scheduled');
-  });
-
-  // ============================================================
-  // Test: Handles empty HTML (no units found)
-  // ============================================================
-  it('should handle HTML with no units gracefully', async () => {
-    const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
-
-    const result = await runScrape(db, {
-      jobId: 'test-empty-html',
-      htmlContent: emptyHtml
-    });
-
-    expect(result.status).toBe('success');
-    expect(result.unitsProcessed).toBe(0);
-    expect(result.errors).toContain('No units found in HTML');
-  });
-});
-
-// ============================================================
-// Test: sanitizeError - Error message sanitization
-// ============================================================
-describe('sanitizeError', () => {
-  let sanitizeError;
-
-  beforeAll(() => {
-    ({ sanitizeError } = require('../../services/scraperService'));
-  });
-
-  // ----------------------------------------------------------
-  // File path sanitization
-  // ----------------------------------------------------------
-  describe('file path sanitization', () => {
-    it('should remove Unix absolute file paths from error messages', () => {
-      const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toMatch(/\/home\/user/);
-      expect(sanitized.message).toContain('ENOENT');
-    });
-
-    it('should remove /var paths from error messages', () => {
-      const error = new Error('Failed to read /var/app/current/data/secrets.yml');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toMatch(/\/var\/app/);
-      expect(sanitized.message).toContain('Failed to read');
-    });
-
-    it('should remove Windows-style file paths from error messages', () => {
-      const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toMatch(/C:\\Users/);
-      expect(sanitized.message).toContain('Cannot find module');
-    });
-
-    it('should remove /tmp and /usr paths from error messages', () => {
-      const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toMatch(/\/tmp\//);
-      expect(sanitized.message).not.toMatch(/\/usr\//);
-    });
-  });
-
-  // ----------------------------------------------------------
-  // MongoDB connection string sanitization
-  // ----------------------------------------------------------
-  describe('connection string sanitization', () => {
-    it('should redact mongodb:// connection strings', () => {
-      const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toContain('s3cretP4ss');
-      expect(sanitized.message).not.toContain('admin:');
-      expect(sanitized.message).toContain('Connection failed');
-      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
-    });
-
-    it('should redact mongodb+srv:// connection strings', () => {
-      const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toContain('password123');
-      expect(sanitized.message).not.toContain('user:');
-      expect(sanitized.message).toContain('Timeout connecting to');
-      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
-    });
-
-    it('should redact connection string without credentials', () => {
-      const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
-      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
-    });
-  });
-
-  // ----------------------------------------------------------
-  // Credential / secret sanitization
-  // ----------------------------------------------------------
-  describe('credential sanitization', () => {
-    it('should redact common environment variable patterns', () => {
-      const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toContain('sk-abc123xyz');
-      expect(sanitized.message).not.toContain('bearer-9876');
-    });
-
-    it('should redact password patterns', () => {
-      const error = new Error('Auth failed with password=MyS3cret!');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).not.toContain('MyS3cret!');
-    });
-  });
-
-  // ----------------------------------------------------------
-  // Error type preservation
-  // ----------------------------------------------------------
-  describe('error type preservation', () => {
-    it('should preserve the error type/name', () => {
-      const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.name).toBe('TypeError');
-    });
-
-    it('should preserve custom error names', () => {
-      const error = new Error('Timeout at /var/app/scraper.js:100');
-      error.name = 'TimeoutError';
-      const sanitized = sanitizeError(error);
-      expect(sanitized.name).toBe('TimeoutError');
-    });
-
-    it('should preserve error name for RangeError', () => {
-      const error = new RangeError('Maximum call stack size exceeded');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.name).toBe('RangeError');
-    });
-  });
-
-  // ----------------------------------------------------------
-  // General description preserved for debugging
-  // ----------------------------------------------------------
-  describe('general description preservation', () => {
-    it('should preserve a useful general description', () => {
-      const error = new Error('Network timeout after 30000ms');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).toContain('Network timeout after 30000ms');
-    });
-
-    it('should preserve error description when no sensitive data present', () => {
-      const error = new Error('Request failed with status code 500');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).toBe('Request failed with status code 500');
-    });
-
-    it('should return a useful message even after heavy sanitization', () => {
-      const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
-      const sanitized = sanitizeError(error);
-      expect(sanitized.message).toContain('ECONNREFUSED');
-      expect(sanitized.message.length).toBeGreaterThan(5);
-    });
-  });
-
-  // ----------------------------------------------------------
-  // Stack trace sanitization
-  // ----------------------------------------------------------
-  describe('stack trace removal', () => {
-    it('should remove file paths from stack traces', () => {
-      const error = new Error('Something failed');
-      error.stack = 'Error: Something failed\n    at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n    at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
-      const sanitized = sanitizeError(error);
-      expect(sanitized.stack).not.toMatch(/\/home\/user/);
-      expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
-    });
-
-    it('should handle errors without stack trace', () => {
-      const error = new Error('No stack');
-      error.stack = undefined;
-      const sanitized = sanitizeError(error);
-      expect(sanitized.stack).toBeUndefined();
-    });
-  });
-
-  // ----------------------------------------------------------
-  // Integration: sanitization applied before recordScraperRun()
-  // ----------------------------------------------------------
-  describe('sanitization before recordScraperRun()', () => {
-    it('should store sanitized error message in scraper_runs on failure', async () => {
-      const html = createSampleHtml(['SANITIZE-A']);
-
-      // Create a db proxy that throws an error containing sensitive info
-      const sensitiveError = new Error(
-        'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
-      );
-      sensitiveError.name = 'MongoServerError';
-
-      const faultyDb = {
-        collection: (name) => {
-          const realCollection = db.collection(name);
-          if (name === 'units_scraper') {
-            return new Proxy(realCollection, {
-              get(target, prop) {
-                if (prop === 'bulkWrite') {
-                  return async () => { throw sensitiveError; };
-                }
-                const value = target[prop];
-                if (typeof value === 'function') {
-                  return value.bind(target);
-                }
-                return value;
-              }
-            });
-          }
-          return realCollection;
-        }
-      };
-
-      const result = await runScrape(faultyDb, {
-        jobId: 'test-sanitized-error',
-        htmlContent: html
-      });
-
-      expect(result.status).toBe('failed');
-
-      // Verify the error stored in result.errors is sanitized
-      const errorMsg = result.errors[0];
-      expect(errorMsg).not.toContain('SuperSecret');
-      expect(errorMsg).not.toContain('admin:');
-      expect(errorMsg).not.toContain('/home/deploy/');
-      expect(errorMsg).toContain('MongoServerError');
-
-      // Verify the error stored in scraper_runs is sanitized
-      const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
-      expect(runRecord).toBeTruthy();
-      expect(runRecord.status).toBe('failed');
-      const storedError = runRecord.errors[0];
-      expect(storedError).not.toContain('SuperSecret');
-      expect(storedError).not.toContain('admin:');
-      expect(storedError).not.toContain('/home/deploy/');
-    });
-  });
-});
-
-// ============================================================
-// Test: createScraperIndexes
-// ============================================================
-describe('createScraperIndexes', () => {
-  const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
-
-  beforeEach(() => {
-    mockLogger.info.mockClear();
-    mockLogger.warn.mockClear();
-    mockLogger.error.mockClear();
-  });
-
-  it('should create the status_startedAt compound index', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    const indexes = await db.collection('scraper_runs').indexes();
-    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
-    expect(statusIndex).toBeTruthy();
-    expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
-  });
-
-  it('should create the startedAt_desc index', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    const indexes = await db.collection('scraper_runs').indexes();
-    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
-    expect(startedAtIndex).toBeTruthy();
-    expect(startedAtIndex.key).toEqual({ startedAt: -1 });
-  });
-
-  it('should log success after creating indexes', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
-  });
-
-  it('should be idempotent (no error on second call)', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    // Calling again should not throw
-    await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
-
-    // Indexes should still exist
-    const indexes = await db.collection('scraper_runs').indexes();
-    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
-    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
-    expect(statusIndex).toBeTruthy();
-    expect(startedAtIndex).toBeTruthy();
-  });
-
-  it('should handle database errors gracefully', async () => {
-    // Pass an object that will throw when collection() is called
-    const faultyDb = {
-      collection: () => {
-        throw new Error('Connection lost');
-      }
-    };
-
-    await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
-  });
-});

__tests__/scraper/serverIntegration.test.js

@@ -1,177 +0,0 @@
-/**
- * Tests for scraper integration in server.js startup
- *
- * Covers:
- * - Server initializes scraper scheduler on startup
- * - Server does not crash if scheduler init fails
- * - Indexes are created before scheduler starts
- * - Scheduler not initialized when SCRAPER_ENABLED=false
- *
- * Strategy: Instead of importing server.js directly (which starts Express and
- * connects to MongoDB), we replicate the scraper initialization logic from
- * connectToMongoDB() and test it with mocked dependencies. This verifies the
- * integration contract without needing a full server boot.
- */
-
-const { MongoClient } = require('mongodb');
-
-describe('Server Scraper Integration', () => {
-  let connection;
-  let db;
-
-  beforeAll(async () => {
-    const uri = process.env.MONGO_URI;
-    connection = await MongoClient.connect(uri);
-    db = connection.db('apartments_server_integration_test');
-  });
-
-  afterAll(async () => {
-    if (connection) {
-      await connection.close();
-    }
-  });
-
-  describe('scraper initialization during startup', () => {
-    let mockCreateScraperIndexes;
-    let mockInitializeScheduler;
-    let mockCreateLogger;
-    let mockLogger;
-
-    beforeEach(() => {
-      mockLogger = {
-        info: jest.fn(),
-        warn: jest.fn(),
-        error: jest.fn(),
-        debug: jest.fn()
-      };
-      mockCreateLogger = jest.fn(() => mockLogger);
-      mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
-      mockInitializeScheduler = jest.fn();
-    });
-
-    /**
-     * Replicates the scraper initialization logic from server.js connectToMongoDB().
-     * This is the exact pattern used in production:
-     *
-     *   try {
-     *     const scraperLogger = createLogger('server-init');
-     *     await createScraperIndexes(db, scraperLogger);
-     *     if (scraperConfig.SCRAPER_ENABLED) {
-     *       initializeScheduler(db);
-     *     }
-     *   } catch (error) {
-     *     console.error('Scraper initialization failed (server continues):', error.message);
-     *   }
-     */
-    async function runScraperInit(testDb, config) {
-      try {
-        const scraperLogger = mockCreateLogger('server-init');
-        await mockCreateScraperIndexes(testDb, scraperLogger);
-        if (config.SCRAPER_ENABLED) {
-          mockInitializeScheduler(testDb);
-        }
-        return { success: true };
-      } catch (error) {
-        return { success: false, error: error.message };
-      }
-    }
-
-    it('should initialize scraper scheduler on startup when enabled', async () => {
-      const config = { SCRAPER_ENABLED: true };
-
-      const result = await runScraperInit(db, config);
-
-      expect(result.success).toBe(true);
-      expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
-      expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
-    });
-
-    it('should not crash if createScraperIndexes throws an error', async () => {
-      mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
-      const config = { SCRAPER_ENABLED: true };
-
-      const result = await runScraperInit(db, config);
-
-      // The server should continue (error is caught, not thrown)
-      expect(result.success).toBe(false);
-      expect(result.error).toBe('Index creation failed');
-      // initializeScheduler should NOT have been called since the error occurred before it
-      expect(mockInitializeScheduler).not.toHaveBeenCalled();
-    });
-
-    it('should create indexes before initializing the scheduler', async () => {
-      const callOrder = [];
-      mockCreateScraperIndexes.mockImplementation(async () => {
-        callOrder.push('createScraperIndexes');
-      });
-      mockInitializeScheduler.mockImplementation(() => {
-        callOrder.push('initializeScheduler');
-      });
-
-      const config = { SCRAPER_ENABLED: true };
-      await runScraperInit(db, config);
-
-      expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
-    });
-
-    it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
-      const config = { SCRAPER_ENABLED: false };
-
-      const result = await runScraperInit(db, config);
-
-      expect(result.success).toBe(true);
-      expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
-      expect(mockInitializeScheduler).not.toHaveBeenCalled();
-    });
-  });
-
-  describe('server.js imports and exports verification', () => {
-    it('should be able to import createScraperIndexes from scraperService', () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      expect(typeof createScraperIndexes).toBe('function');
-    });
-
-    it('should be able to import initializeScheduler from scraperJob', () => {
-      const { initializeScheduler } = require('../../jobs/scraperJob');
-      expect(typeof initializeScheduler).toBe('function');
-    });
-
-    it('should be able to import registerSignalHandlers from scraperJob', () => {
-      const { registerSignalHandlers } = require('../../jobs/scraperJob');
-      expect(typeof registerSignalHandlers).toBe('function');
-    });
-
-    it('should be able to import SCRAPER_ENABLED from scraper config', () => {
-      const scraperConfig = require('../../config/scraper');
-      expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
-    });
-  });
-
-  describe('createScraperIndexes error handling with real db', () => {
-    it('should handle index creation gracefully with a valid db', async () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      const { createLogger } = require('../../services/scraperLogger');
-      const logger = createLogger('test-server-init');
-
-      // Should not throw - creates indexes on the test db
-      await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
-    });
-
-    it('should throw when given an invalid db object', async () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
-
-      // Passing null should throw (simulates what the try/catch in server.js protects against)
-      await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
-    });
-  });
-
-  describe('registerSignalHandlers', () => {
-    it('should register SIGTERM and SIGINT handlers without throwing', () => {
-      const { registerSignalHandlers } = require('../../jobs/scraperJob');
-
-      // Should not throw when called
-      expect(() => registerSignalHandlers()).not.toThrow();
-    });
-  });
-});

__tests__/scraper/updateDailySummary.test.js

@@ -1,561 +0,0 @@
-/**
- * Tests for updateDailySummary() daily summary aggregation
- *
- * Covers:
- * - Upsert by date key (YYYY-MM-DD)
- * - Summary includes new_units, rented_units, stale_units arrays
- * - Count fields are calculated correctly
- * - net_change = newUnits.length - rentedUnits.length
- * - total_available_today is set correctly
- * - total_available_yesterday fetched from previous day
- * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
- * - turnover_rate is 0 when yesterdayTotal is 0
- * - Error handling when updateOne fails
- * - Logging of summary update details
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-// Will require updateDailySummary after implementation
-let updateDailySummary;
-
-let mongoServer;
-let client;
-let db;
-
-const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
-
-// Mock logger for capturing log calls
-function createMockLogger() {
-  return {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn()
-  };
-}
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  ({ updateDailySummary } = require('../../services/scraperService'));
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean collections before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-});
-
-describe('updateDailySummary', () => {
-  // ---------------------------------------------------------------
-  // 1. Upsert by date key
-  // ---------------------------------------------------------------
-  describe('upsert by date key', () => {
-    it('should insert a new summary when no document exists for the date', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 2,
-        totalAvailable: 15
-      };
-
-      const result = await updateDailySummary(db, summaryData, logger);
-
-      // Should have upserted (created new)
-      expect(result.upsertedCount).toBe(1);
-
-      // Verify the document in the database
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc).not.toBeNull();
-      expect(doc.date).toBe('2026-02-05');
-    });
-
-    it('should update an existing summary for the same date (idempotent upsert)', async () => {
-      const logger = createMockLogger();
-
-      // Insert initial summary
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-02-05',
-        new_units: ['A101'],
-        rented_units: [],
-        total_available_today: 10
-      });
-
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102', 'A103'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 1,
-        totalAvailable: 12
-      };
-
-      const result = await updateDailySummary(db, summaryData, logger);
-
-      // Should have matched and modified (not upserted)
-      expect(result.matchedCount).toBe(1);
-
-      // Verify updated values
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
-      expect(doc.total_available_today).toBe(12);
-
-      // Verify only one document exists for this date
-      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
-      expect(count).toBe(1);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. Summary includes new_units, rented_units, stale_units arrays
-  // ---------------------------------------------------------------
-  describe('summary arrays', () => {
-    it('should store new_units, rented_units, and stale_units arrays', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102'],
-        rentedUnits: ['B201', 'B202', 'B203'],
-        staleUnitsCount: 3,
-        totalAvailable: 20
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.new_units).toEqual(['A101', 'A102']);
-      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
-      expect(doc.stale_units).toEqual([]);
-    });
-
-    it('should handle empty arrays for new_units and rented_units', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 25
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.new_units).toEqual([]);
-      expect(doc.rented_units).toEqual([]);
-      expect(doc.stale_units).toEqual([]);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. Count fields are calculated correctly
-  // ---------------------------------------------------------------
-  describe('count fields', () => {
-    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102', 'A103'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 5,
-        totalAvailable: 30
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.new_units_count).toBe(3);
-      expect(doc.rented_units_count).toBe(1);
-      expect(doc.stale_units_count).toBe(5);
-    });
-
-    it('should set all count fields to 0 when arrays are empty', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.new_units_count).toBe(0);
-      expect(doc.rented_units_count).toBe(0);
-      expect(doc.stale_units_count).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. net_change = newUnits.length - rentedUnits.length
-  // ---------------------------------------------------------------
-  describe('net_change calculation', () => {
-    it('should calculate positive net_change when more units added than rented', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102', 'A103'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 0,
-        totalAvailable: 20
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.net_change).toBe(2); // 3 - 1 = 2
-    });
-
-    it('should calculate negative net_change when more units rented than added', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
-    });
-
-    it('should calculate zero net_change when equal new and rented', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102'],
-        rentedUnits: ['B201', 'B202'],
-        staleUnitsCount: 0,
-        totalAvailable: 15
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.net_change).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. total_available_today is set correctly
-  // ---------------------------------------------------------------
-  describe('total_available_today', () => {
-    it('should store the totalAvailable value as total_available_today', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 42
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.total_available_today).toBe(42);
-    });
-
-    it('should handle totalAvailable of 0', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 0,
-        totalAvailable: 0
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.total_available_today).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. total_available_yesterday fetched from previous day
-  // ---------------------------------------------------------------
-  describe('total_available_yesterday', () => {
-    it('should fetch total_available_today from previous day summary', async () => {
-      const logger = createMockLogger();
-
-      // Insert yesterday's summary
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-02-04',
-        total_available_today: 35,
-        new_units: [],
-        rented_units: []
-      });
-
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 0,
-        totalAvailable: 35
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.total_available_yesterday).toBe(35);
-    });
-
-    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 20
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.total_available_yesterday).toBe(0);
-    });
-
-    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
-      const logger = createMockLogger();
-
-      // Insert March 31 summary
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-03-31',
-        total_available_today: 50
-      });
-
-      const summaryData = {
-        date: '2026-04-01',
-        newUnits: [],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 48
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
-      expect(doc.total_available_yesterday).toBe(50);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
-  // ---------------------------------------------------------------
-  describe('turnover_rate calculation', () => {
-    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
-      const logger = createMockLogger();
-
-      // Insert yesterday's summary with 50 total available
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-02-04',
-        total_available_today: 50
-      });
-
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
-        staleUnitsCount: 0,
-        totalAvailable: 46
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      // 5 rented / 50 yesterday = 10%
-      expect(doc.turnover_rate).toBe(10);
-    });
-
-    it('should round turnover_rate to 2 decimal places', async () => {
-      const logger = createMockLogger();
-
-      // Insert yesterday's summary with 30 total available
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-02-04',
-        total_available_today: 30
-      });
-
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: ['B201', 'B202'],
-        staleUnitsCount: 0,
-        totalAvailable: 28
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
-      expect(doc.turnover_rate).toBe(6.67);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. turnover_rate is 0 when yesterdayTotal is 0
-  // ---------------------------------------------------------------
-  describe('turnover_rate edge cases', () => {
-    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 0,
-        totalAvailable: 15
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.turnover_rate).toBe(0);
-    });
-
-    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
-      const logger = createMockLogger();
-
-      // Insert yesterday's summary with 0 total available
-      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
-        date: '2026-02-04',
-        total_available_today: 0
-      });
-
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.turnover_rate).toBe(0);
-    });
-
-    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.turnover_rate).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 9. Error handling when updateOne fails
-  // ---------------------------------------------------------------
-  describe('error handling', () => {
-    it('should throw and log error when updateOne fails', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101'],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      // Create a mock db that throws on updateOne
-      const mockCollection = {
-        findOne: jest.fn().mockResolvedValue(null),
-        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await expect(
-        updateDailySummary(mockDb, summaryData, logger)
-      ).rejects.toThrow('Write concern timeout');
-
-      // Verify error was logged
-      expect(logger.error).toHaveBeenCalledWith(
-        'Failed to update daily summary',
-        expect.objectContaining({
-          errorType: 'Error',
-          errorMessage: 'Write concern timeout'
-        })
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 10. Logging of summary update details
-  // ---------------------------------------------------------------
-  describe('logging', () => {
-    it('should log summary update details on success', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: ['A101', 'A102'],
-        rentedUnits: ['B201'],
-        staleUnitsCount: 1,
-        totalAvailable: 18
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Daily summary updated',
-        expect.objectContaining({
-          date: '2026-02-05',
-          newUnits: 2,
-          rentedUnits: 1,
-          totalAvailable: 18
-        })
-      );
-    });
-
-    it('should include timestamp field in the stored summary', async () => {
-      const logger = createMockLogger();
-      const summaryData = {
-        date: '2026-02-05',
-        newUnits: [],
-        rentedUnits: [],
-        staleUnitsCount: 0,
-        totalAvailable: 10
-      };
-
-      await updateDailySummary(db, summaryData, logger);
-
-      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
-      expect(doc.timestamp).toBeDefined();
-      expect(typeof doc.timestamp).toBe('string');
-      // Verify it is a valid ISO timestamp
-      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
-    });
-  });
-});

__tests__/scraper/upsertUnits.test.js

@@ -1,498 +0,0 @@
-/**
- * Tests for upsertUnits() bulk operation
- *
- * Covers:
- * - bulkWrite is called with correct updateOne operations
- * - upsert: true is set for all operations
- * - $set updates last_scraped and data_source fields
- * - $setOnInsert sets first_seen for new units
- * - Empty units array logs warning and returns early
- * - ordered: false is set for parallel execution
- * - Error handling when bulkWrite fails
- * - Logging of matched, modified, and upserted counts
- */
-
-const { MongoClient } = require('mongodb');
-const { MongoMemoryServer } = require('mongodb-memory-server');
-
-// We will require upsertUnits after implementation
-let upsertUnits;
-
-let mongoServer;
-let client;
-let db;
-
-// Mock logger for capturing log calls
-function createMockLogger() {
-  return {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn()
-  };
-}
-
-beforeAll(async () => {
-  mongoServer = await MongoMemoryServer.create();
-  const uri = mongoServer.getUri();
-  client = new MongoClient(uri);
-  await client.connect();
-  db = client.db('test_apartments');
-
-  // Dynamically require to pick up implementation
-  ({ upsertUnits } = require('../../services/scraperService'));
-});
-
-afterAll(async () => {
-  if (client) await client.close();
-  if (mongoServer) await mongoServer.stop();
-});
-
-beforeEach(async () => {
-  // Clean the units collection before each test
-  const collections = await db.listCollections().toArray();
-  for (const col of collections) {
-    await db.collection(col.name).deleteMany({});
-  }
-});
-
-describe('upsertUnits', () => {
-  const UNITS_COLLECTION = 'units_scraper';
-
-  // ---------------------------------------------------------------
-  // 1. bulkWrite is called with correct updateOne operations
-  // ---------------------------------------------------------------
-  describe('bulkWrite operations', () => {
-    it('should create updateOne operations for each unit', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
-        { unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
-      ];
-
-      const result = await upsertUnits(db, units, logger);
-
-      // Both units should be upserted (new inserts)
-      expect(result.upsertedCount).toBe(2);
-
-      // Verify documents exist in the collection
-      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(2);
-
-      const unitCodes = docs.map(d => d.unit_code).sort();
-      expect(unitCodes).toEqual(['A101', 'B202']);
-    });
-
-    it('should filter by unit_code as the unique key', async () => {
-      const logger = createMockLogger();
-
-      // Insert a unit first
-      const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
-      await upsertUnits(db, units, logger);
-
-      // Upsert again with updated price
-      const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
-      const result = await upsertUnits(db, updatedUnits, logger);
-
-      // Should match and modify, not insert new
-      expect(result.matchedCount).toBe(1);
-      expect(result.upsertedCount).toBe(0);
-
-      // Verify only one document exists
-      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
-      expect(docs).toHaveLength(1);
-      expect(docs[0].price).toBe(1400);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 2. upsert: true is set for all operations
-  // ---------------------------------------------------------------
-  describe('upsert behavior', () => {
-    it('should insert new units when they do not exist (upsert: true)', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'NEW-001', price: 1100 },
-        { unit_code: 'NEW-002', price: 1200 },
-        { unit_code: 'NEW-003', price: 1300 }
-      ];
-
-      const result = await upsertUnits(db, units, logger);
-
-      expect(result.upsertedCount).toBe(3);
-
-      // Verify all three are in the database
-      const count = await db.collection(UNITS_COLLECTION).countDocuments();
-      expect(count).toBe(3);
-    });
-
-    it('should update existing units without creating duplicates', async () => {
-      const logger = createMockLogger();
-
-      // First insert
-      await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
-
-      // Second upsert with same unit_code
-      await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
-
-      const count = await db.collection(UNITS_COLLECTION).countDocuments();
-      expect(count).toBe(1);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 3. $set updates last_scraped and data_source fields
-  // ---------------------------------------------------------------
-  describe('$set fields', () => {
-    it('should set last_scraped timestamp on every upsert', async () => {
-      const logger = createMockLogger();
-      const beforeTime = new Date().toISOString();
-
-      const units = [{ unit_code: 'T100', price: 1000 }];
-      await upsertUnits(db, units, logger);
-
-      const afterTime = new Date().toISOString();
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
-      expect(doc.last_scraped).toBeDefined();
-      expect(typeof doc.last_scraped).toBe('string');
-      // The last_scraped should be between before and after timestamps
-      expect(doc.last_scraped >= beforeTime).toBe(true);
-      expect(doc.last_scraped <= afterTime).toBe(true);
-    });
-
-    it('should set data_source to "web_scraper"', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'T200', price: 1100 }];
-
-      await upsertUnits(db, units, logger);
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
-      expect(doc.data_source).toBe('web_scraper');
-    });
-
-    it('should update last_scraped on subsequent upserts', async () => {
-      const logger = createMockLogger();
-
-      // First insert
-      await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
-      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
-      const firstScraped = doc1.last_scraped;
-
-      // Small delay to ensure different timestamp
-      await new Promise(resolve => setTimeout(resolve, 10));
-
-      // Second upsert
-      await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
-      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
-      const secondScraped = doc2.last_scraped;
-
-      expect(secondScraped > firstScraped).toBe(true);
-    });
-
-    it('should spread all unit fields into the $set operation', async () => {
-      const logger = createMockLogger();
-      const units = [{
-        unit_code: 'T400',
-        price: 1500,
-        bed_count: 2,
-        bath_count: 1.5,
-        area: 900,
-        floor: 3,
-        plan_name: 'Luxury',
-        community: 'Tower A'
-      }];
-
-      await upsertUnits(db, units, logger);
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
-      expect(doc.price).toBe(1500);
-      expect(doc.bed_count).toBe(2);
-      expect(doc.bath_count).toBe(1.5);
-      expect(doc.area).toBe(900);
-      expect(doc.floor).toBe(3);
-      expect(doc.plan_name).toBe('Luxury');
-      expect(doc.community).toBe('Tower A');
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 4. $setOnInsert sets first_seen for new units
-  // ---------------------------------------------------------------
-  describe('$setOnInsert for first_seen', () => {
-    it('should set first_seen on initial insert', async () => {
-      const logger = createMockLogger();
-      const beforeTime = new Date().toISOString();
-
-      await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
-
-      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
-      expect(doc.first_seen).toBeDefined();
-      expect(typeof doc.first_seen).toBe('string');
-      expect(doc.first_seen >= beforeTime).toBe(true);
-    });
-
-    it('should NOT update first_seen on subsequent upserts', async () => {
-      const logger = createMockLogger();
-
-      // First insert
-      await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
-      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
-      const originalFirstSeen = doc1.first_seen;
-
-      // Small delay to ensure different timestamp
-      await new Promise(resolve => setTimeout(resolve, 10));
-
-      // Second upsert (update)
-      await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
-      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
-
-      // first_seen should remain unchanged
-      expect(doc2.first_seen).toBe(originalFirstSeen);
-    });
-
-    it('should set different first_seen for different units inserted at different times', async () => {
-      const logger = createMockLogger();
-
-      await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
-      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
-
-      await new Promise(resolve => setTimeout(resolve, 10));
-
-      await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
-      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
-
-      // Both should have first_seen, but different values
-      expect(doc1.first_seen).toBeDefined();
-      expect(doc2.first_seen).toBeDefined();
-      expect(doc2.first_seen > doc1.first_seen).toBe(true);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 5. Empty units array logs warning and returns early
-  // ---------------------------------------------------------------
-  describe('empty units array', () => {
-    it('should log a warning when units array is empty', async () => {
-      const logger = createMockLogger();
-
-      await upsertUnits(db, [], logger);
-
-      expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
-    });
-
-    it('should return early with zero counts for empty array', async () => {
-      const logger = createMockLogger();
-
-      const result = await upsertUnits(db, [], logger);
-
-      expect(result.modifiedCount).toBe(0);
-      expect(result.upsertedCount).toBe(0);
-    });
-
-    it('should not perform any database writes for empty array', async () => {
-      const logger = createMockLogger();
-
-      await upsertUnits(db, [], logger);
-
-      const count = await db.collection(UNITS_COLLECTION).countDocuments();
-      expect(count).toBe(0);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 6. ordered: false is set for parallel execution
-  // ---------------------------------------------------------------
-  describe('ordered: false for parallel execution', () => {
-    it('should successfully process all units even if one has a conflict', async () => {
-      const logger = createMockLogger();
-
-      // Insert multiple units - ordered: false means all operations execute
-      // even if some fail. We verify by checking all valid units are present.
-      const units = [
-        { unit_code: 'P100', price: 1000 },
-        { unit_code: 'P200', price: 1100 },
-        { unit_code: 'P300', price: 1200 },
-        { unit_code: 'P400', price: 1300 },
-        { unit_code: 'P500', price: 1400 }
-      ];
-
-      const result = await upsertUnits(db, units, logger);
-
-      expect(result.upsertedCount).toBe(5);
-
-      // All 5 units should be in the database
-      const count = await db.collection(UNITS_COLLECTION).countDocuments();
-      expect(count).toBe(5);
-    });
-
-    it('should handle a mix of new inserts and existing updates', async () => {
-      const logger = createMockLogger();
-
-      // First, insert some units
-      await upsertUnits(db, [
-        { unit_code: 'M100', price: 900 },
-        { unit_code: 'M200', price: 1000 }
-      ], logger);
-
-      // Now upsert a mix of existing and new units
-      const mixedUnits = [
-        { unit_code: 'M100', price: 950 },  // existing - update
-        { unit_code: 'M200', price: 1050 }, // existing - update
-        { unit_code: 'M300', price: 1100 }, // new - insert
-      ];
-
-      const result = await upsertUnits(db, mixedUnits, logger);
-
-      expect(result.matchedCount).toBe(2);
-      expect(result.upsertedCount).toBe(1);
-
-      // Total should be 3 documents
-      const count = await db.collection(UNITS_COLLECTION).countDocuments();
-      expect(count).toBe(3);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 7. Error handling when bulkWrite fails
-  // ---------------------------------------------------------------
-  describe('error handling', () => {
-    it('should throw error when bulkWrite fails', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'E100', price: 1000 }];
-
-      // Create a mock db that throws on bulkWrite
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
-    });
-
-    it('should log error details when bulkWrite fails', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'E200', price: 1000 }];
-
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      try {
-        await upsertUnits(mockDb, units, logger);
-      } catch (e) {
-        // Expected to throw
-      }
-
-      expect(logger.error).toHaveBeenCalledWith(
-        'Failed to upsert units',
-        expect.objectContaining({
-          errorType: 'Error',
-          errorMessage: 'Write concern timeout'
-        })
-      );
-    });
-
-    it('should re-throw the original error', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'E300', price: 1000 }];
-
-      const originalError = new TypeError('Invalid operation');
-      const mockCollection = {
-        bulkWrite: jest.fn().mockRejectedValue(originalError)
-      };
-      const mockDb = {
-        collection: jest.fn().mockReturnValue(mockCollection)
-      };
-
-      await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. Logging of matched, modified, and upserted counts
-  // ---------------------------------------------------------------
-  describe('logging of result counts', () => {
-    it('should log matched, modified, and upserted counts on success', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'L100', price: 1000 },
-        { unit_code: 'L200', price: 1100 }
-      ];
-
-      await upsertUnits(db, units, logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Units upserted',
-        expect.objectContaining({
-          matched: expect.any(Number),
-          modified: expect.any(Number),
-          upserted: expect.any(Number)
-        })
-      );
-    });
-
-    it('should log correct counts for new inserts', async () => {
-      const logger = createMockLogger();
-      const units = [
-        { unit_code: 'L300', price: 1000 },
-        { unit_code: 'L400', price: 1100 }
-      ];
-
-      await upsertUnits(db, units, logger);
-
-      // For new inserts: matched=0, modified=0, upserted=2
-      expect(logger.info).toHaveBeenCalledWith(
-        'Units upserted',
-        expect.objectContaining({
-          matched: 0,
-          modified: 0,
-          upserted: 2
-        })
-      );
-    });
-
-    it('should log correct counts for updates to existing units', async () => {
-      const logger = createMockLogger();
-
-      // First insert
-      await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
-
-      // Reset mock to capture only the second call
-      logger.info.mockClear();
-
-      // Update existing
-      await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
-
-      expect(logger.info).toHaveBeenCalledWith(
-        'Units upserted',
-        expect.objectContaining({
-          matched: 1,
-          modified: 1,
-          upserted: 0
-        })
-      );
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 9. Return value
-  // ---------------------------------------------------------------
-  describe('return value', () => {
-    it('should return the bulkWrite result object', async () => {
-      const logger = createMockLogger();
-      const units = [{ unit_code: 'R100', price: 1000 }];
-
-      const result = await upsertUnits(db, units, logger);
-
-      // bulkWrite result should have these standard properties
-      expect(result).toHaveProperty('matchedCount');
-      expect(result).toHaveProperty('modifiedCount');
-      expect(result).toHaveProperty('upsertedCount');
-    });
-  });
-});

config/scraper.js

@@ -1,41 +0,0 @@
-/**
- * Scraper Configuration Module
- *
- * Contains all constants and settings for the apartment price scraper.
- * All values can be overridden via environment variables for deployment flexibility.
- *
- * See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
- */
-
-module.exports = {
-  // Target URL for apartment listings
-  TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
-
-  // Scheduling configuration
-  SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
-  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
-  SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
-
-  // HTTP settings
-  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
-  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
-
-  // Retry configuration for HTTP requests
-  RETRY_CONFIG: {
-    maxRetries: 3,
-    baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
-    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
-  },
-
-  // Graceful shutdown timeout (how long to wait for running job before force-stopping)
-  SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
-
-  // MongoDB collection names - defaults to validation collections for scraper validation;
-  // switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
-  COLLECTIONS: {
-    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper',
-    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper',
-    DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
-    SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
-  }
-};

cosign.pub

@@ -1,4 +0,0 @@
------BEGIN PUBLIC KEY-----
-MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEBOFl4SHzoZVM59+668t2tecTyKM+
-cE4zaOkbFO30u7d2+bjpwPnbYrTutKmAeehYzCAVW/OYea7ML0cW2YCSCQ==
------END PUBLIC KEY-----

docker-compose.yml

@@ -1,6 +1,5 @@
 services:
   apartment-api:
-    image: ${IMAGE:-apartment-api:latest}
     build: .
     container_name: apartment-api
     restart: unless-stopped

eslint.config.mjs

@@ -1,34 +0,0 @@
-import js from '@eslint/js';
-import globals from 'globals';
-
-export default [
-  js.configs.recommended,
-  {
-    languageOptions: {
-      ecmaVersion: 2022,
-      sourceType: 'commonjs',
-      globals: {
-        ...globals.node,
-        ...globals.jest,
-      },
-    },
-    rules: {
-      // Allow unused vars with underscore prefix, and unused function args
-      'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
-      'no-console': 'off',
-      // Formatting rules as warnings (won't fail CI, but will be reported)
-      'semi': ['warn', 'always'],
-      'no-multiple-empty-lines': ['warn', { max: 2 }],
-      'eol-last': ['warn', 'always'],
-      'no-trailing-spaces': 'warn',
-    },
-  },
-  {
-    ignores: [
-      'node_modules/**',
-      'coverage/**',
-      '*.min.js',
-      'eslint.config.mjs',
-    ],
-  },
-];

jobs/scraperJob.js

@@ -1,275 +0,0 @@
-const cron = require('node-cron');
-const crypto = require('crypto');
-const config = require('../config/scraper');
-const { runScrape } = require('../services/scraperService');
-const { createLogger } = require('../services/scraperLogger');
-
-// In-process mutex state
-let isRunning = false;
-let currentJobId = null;
-
-// Scheduler state
-let scheduledJob = null;
-
-// Shutdown state
-let shuttingDown = false;
-let runningJobPromise = null;
-
-/**
- * Check if scraper is currently running
- * @returns {boolean}
- */
-function isScraperRunning() {
-  return isRunning;
-}
-
-/**
- * Get current job ID if running
- * @returns {string|null}
- */
-function getCurrentJobId() {
-  return currentJobId;
-}
-
-/**
- * Acquire the scraper lock
- * @param {string} jobId - Job ID to set
- * @returns {boolean} True if lock acquired
- */
-function acquireLock(jobId) {
-  if (isRunning || shuttingDown) {
-    return false;
-  }
-  isRunning = true;
-  currentJobId = jobId;
-  return true;
-}
-
-/**
- * Release the scraper lock
- */
-function releaseLock() {
-  isRunning = false;
-  currentJobId = null;
-}
-
-/**
- * Get the configured schedule expression
- * @returns {string} Cron expression or 'disabled'
- */
-function getScheduleExpression() {
-  if (!config.SCRAPER_ENABLED) {
-    return 'disabled';
-  }
-  return config.SCRAPER_SCHEDULE;
-}
-
-/**
- * Calculate next scheduled run time
- * @returns {string|null} ISO timestamp or null if disabled
- */
-function getNextScheduledRun() {
-  if (!config.SCRAPER_ENABLED || !scheduledJob) {
-    return null;
-  }
-
-  const { CronExpressionParser } = require('cron-parser');
-  try {
-    const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
-      tz: config.SCRAPER_TIMEZONE
-    });
-    return interval.next().toISOString();
-  } catch (error) {
-    return null;
-  }
-}
-
-/**
- * Check if schedule runs more frequently than 1 hour
- * @param {string} schedule - Cron expression
- * @returns {boolean} True if schedule is too frequent
- */
-function isScheduleTooFrequent(schedule) {
-  const parts = schedule.trim().split(/\s+/);
-
-  if (parts.length < 5) return false;
-
-  const minuteField = parts[0];
-
-  // If minute field is */N with N < 60, it runs more than once per hour
-  if (/^\*\/\d+$/.test(minuteField)) {
-    const interval = parseInt(minuteField.substring(2), 10);
-    if (interval < 60) return true;
-  }
-
-  // If minute field is *, it runs every minute
-  if (minuteField === '*') return true;
-
-  return false;
-}
-
-/**
- * Initialize the scraper scheduler
- * @param {Db} db - MongoDB database instance
- */
-function initializeScheduler(db) {
-  const logger = createLogger('scheduler');
-
-  // Check if scheduling is enabled
-  if (!config.SCRAPER_ENABLED) {
-    logger.info('Scraper scheduling is disabled');
-    return;
-  }
-
-  // Validate cron expression
-  if (!cron.validate(config.SCRAPER_SCHEDULE)) {
-    logger.error('Invalid cron schedule expression', {
-      schedule: config.SCRAPER_SCHEDULE
-    });
-    logger.warn('Falling back to default schedule: 0 6 * * *');
-    config.SCRAPER_SCHEDULE = '0 6 * * *';
-  }
-
-  // Validate minimum interval (1 hour)
-  if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
-    logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
-      originalSchedule: config.SCRAPER_SCHEDULE
-    });
-    config.SCRAPER_SCHEDULE = '0 * * * *';
-  }
-
-  // Create the scheduled job
-  scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
-    const jobId = crypto.randomUUID();
-    const jobLogger = createLogger(jobId);
-
-    jobLogger.info('Scheduled scrape triggered');
-
-    // Check if already running or shutting down
-    if (!acquireLock(jobId)) {
-      jobLogger.warn('Skipped - scrape already in progress');
-      return;
-    }
-
-    try {
-      const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
-      runningJobPromise = jobExecution;
-      await jobExecution;
-    } catch (error) {
-      jobLogger.error('Scheduled scrape failed', {
-        errorType: error.name,
-        errorMessage: error.message
-      });
-    } finally {
-      runningJobPromise = null;
-      releaseLock();
-    }
-  }, {
-    timezone: config.SCRAPER_TIMEZONE,
-    scheduled: true
-  });
-
-  logger.info('Scraper scheduler initialized', {
-    schedule: config.SCRAPER_SCHEDULE,
-    timezone: config.SCRAPER_TIMEZONE,
-    nextRun: getNextScheduledRun()
-  });
-}
-
-/**
- * Stop the scheduler (for graceful shutdown)
- */
-function stopScheduler() {
-  if (scheduledJob) {
-    scheduledJob.stop();
-    scheduledJob = null;
-  }
-}
-
-/**
- * Check if the scraper is in the process of shutting down
- * @returns {boolean}
- */
-function isShuttingDown() {
-  return shuttingDown;
-}
-
-/**
- * Perform a graceful shutdown of the scraper
- * - Stops the cron scheduler to prevent new jobs
- * - Waits for any running job to complete (with timeout)
- * - Releases the mutex lock
- * - Logs shutdown progress
- * @returns {Promise<void>}
- */
-async function gracefulShutdown() {
-  const logger = createLogger('shutdown');
-
-  logger.info('Shutdown initiated');
-
-  // Mark as shutting down to prevent new jobs
-  shuttingDown = true;
-
-  // Stop the cron scheduler
-  stopScheduler();
-
-  // Wait for running job to complete (with timeout)
-  if (isRunning && runningJobPromise) {
-    logger.info('Waiting for running job to complete', {
-      jobId: currentJobId,
-      timeout: config.SHUTDOWN_TIMEOUT
-    });
-
-    let timeoutHandle;
-    const timeoutPromise = new Promise((resolve) => {
-      timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
-    });
-
-    const result = await Promise.race([
-      runningJobPromise.then(() => 'completed').catch(() => 'completed'),
-      timeoutPromise
-    ]);
-
-    clearTimeout(timeoutHandle);
-
-    if (result === 'timeout') {
-      logger.warn('Shutdown wait for running job timed out', {
-        jobId: currentJobId,
-        timeout: config.SHUTDOWN_TIMEOUT
-      });
-    }
-  }
-
-  // Force release the lock
-  releaseLock();
-
-  logger.info('Shutdown complete');
-}
-
-/**
- * Register process signal handlers for graceful shutdown
- * Listens for SIGTERM and SIGINT signals
- */
-function registerSignalHandlers() {
-  process.on('SIGTERM', () => {
-    gracefulShutdown();
-  });
-
-  process.on('SIGINT', () => {
-    gracefulShutdown();
-  });
-}
-
-module.exports = {
-  isScraperRunning,
-  getCurrentJobId,
-  acquireLock,
-  releaseLock,
-  getScheduleExpression,
-  getNextScheduledRun,
-  initializeScheduler,
-  stopScheduler,
-  isShuttingDown,
-  gracefulShutdown,
-  registerSignalHandlers
-};

package.json

@@ -8,9 +8,7 @@
         "dev": "nodemon server.js",
         "test": "jest",
         "test:watch": "jest --watch",
-        "test:coverage": "jest --coverage",
+        "test:coverage": "jest --coverage"
-        "lint": "eslint .",
-        "lint:fix": "eslint . --fix"
     },
     "keywords": [
         "apartments",
@@ -21,24 +19,17 @@
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
-        "axios": "^1.13.4",
-        "cheerio": "^1.2.0",
         "cookie-parser": "^1.4.7",
         "cors": "^2.8.5",
-        "cron-parser": "^5.5.0",
         "express": "^4.18.2",
         "express-rate-limit": "^7.1.5",
         "jsonwebtoken": "^9.0.3",
         "mongodb": "^6.3.0",
-        "node-cron": "^4.2.1",
         "passport": "^0.7.0",
         "passport-google-oauth20": "^2.0.0",
         "uuid": "^13.0.0"
     },
     "devDependencies": {
-        "@eslint/js": "^9.18.0",
-        "eslint": "^9.18.0",
-        "globals": "^15.14.0",
         "jest": "^30.2.0",
         "mongodb-memory-server": "^11.0.1",
         "nodemon": "^3.0.2",

routes/admin.js

@@ -12,39 +12,6 @@ const { logActivity, ACTIVITY_COLLECTION } = require('../services/activityLogger
 
 const router = express.Router({ strict: true });
 
-// Statistics cache (5-minute TTL)
-const statsCache = new Map();
-const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
-
-/**
- * Get cached value if still valid
- * @param {string} key - Cache key
- * @returns {*} Cached data or null
- */
-function getCached(key) {
-  const cached = statsCache.get(key);
-  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
-    return cached.data;
-  }
-  return null;
-}
-
-/**
- * Set cache value
- * @param {string} key - Cache key
- * @param {*} data - Data to cache
- */
-function setCache(key, data) {
-  statsCache.set(key, { data, timestamp: Date.now() });
-}
-
-/**
- * Clear all cached statistics (for testing)
- */
-function clearStatsCache() {
-  statsCache.clear();
-}
-
 // All admin routes require authentication and admin role
 router.use(requireAuth, requireAdmin);
 
@@ -71,7 +38,7 @@ router.get('/users', async (req, res) => {
       order
     });
 
-    res.json({ data: result });
+    res.json(result);
   } catch (error) {
     console.error('Error fetching users:', error);
     res.status(500).json({ error: 'Failed to fetch users' });
@@ -106,12 +73,10 @@ router.get('/users/:id', async (req, res) => {
       .toArray();
 
     res.json({
-      data: {
       user: {
         ...user,
         recentActivity
       }
-      }
     });
   } catch (error) {
     console.error('Error fetching user details:', error);
@@ -164,10 +129,8 @@ router.patch('/users/:id', async (req, res) => {
     });
 
     res.json({
-      data: {
       user: updatedUser,
       message: isActive ? 'User enabled successfully' : 'User disabled successfully'
-      }
     });
   } catch (error) {
     console.error('Error updating user status:', error);
@@ -200,36 +163,17 @@ router.patch('/users/:id/role', async (req, res) => {
       return res.status(400).json({ error: 'Invalid role. Must be "user" or "admin"' });
     }
 
+    // Check if admin is trying to demote themselves
+    if (req.user._id.toString() === id && role === 'user') {
+      return res.status(400).json({ error: 'Cannot demote yourself from admin' });
+    }
+
     // Check if user exists
     const existingUser = await findById(db, id);
     if (!existingUser) {
       return res.status(404).json({ error: 'User not found' });
     }
 
-    // Check if this would demote the last admin (before self-demotion check)
-    // This is the more specific error when both conditions are true
-    if (existingUser.role === 'admin' && role === 'user') {
-      // Count active admins
-      const activeAdminCount = await db.collection('users').countDocuments({
-        role: 'admin',
-        isActive: true
-      });
-
-      if (activeAdminCount <= 1) {
-        // Customize message if admin is demoting themselves AND they're the last admin
-        const isSelfDemotion = req.user._id.toString() === id;
-        const errorMsg = isSelfDemotion
-          ? 'Cannot demote yourself. You are the last admin and at least one admin must exist.'
-          : 'Cannot demote the last admin. At least one admin must exist.';
-        return res.status(400).json({ error: errorMsg });
-      }
-    }
-
-    // Check if admin is trying to demote themselves (only reached if not last admin)
-    if (req.user._id.toString() === id && role === 'user') {
-      return res.status(400).json({ error: 'Cannot demote yourself from admin' });
-    }
-
     // Update user role
     const updatedUser = await updateUserRole(db, id, role);
 
@@ -251,10 +195,8 @@ router.patch('/users/:id/role', async (req, res) => {
       : 'User demoted to user successfully';
 
     res.json({
-      data: {
       user: updatedUser,
       message
-      }
     });
   } catch (error) {
     console.error('Error updating user role:', error);
@@ -303,7 +245,6 @@ router.get('/users/:id/activity', async (req, res) => {
     ]);
 
     res.json({
-      data: {
       activities,
       pagination: {
         page,
@@ -311,7 +252,6 @@ router.get('/users/:id/activity', async (req, res) => {
         total,
         pages: Math.ceil(total / limit) || 0
       }
-      }
     });
   } catch (error) {
     console.error('Error fetching user activity:', error);
@@ -440,7 +380,6 @@ router.get('/activity', async (req, res) => {
     const activities = await collection.aggregate(pipeline).toArray();
 
     res.json({
-      data: {
       activities,
       pagination: {
         page: validatedPage,
@@ -448,7 +387,6 @@ router.get('/activity', async (req, res) => {
         total,
         pages: Math.ceil(total / validatedLimit) || 0
       }
-      }
     });
   } catch (error) {
     console.error('Error fetching activity:', error);
@@ -584,794 +522,4 @@ router.get('/activity/export', async (req, res) => {
   }
 });
 
-// ============================================================
-// Statistics Endpoints (Phase 3)
-// ============================================================
-
-/**
- * GET /api/admin/stats/overview
- * Returns summary metrics for the admin dashboard
- * Cached for 5 minutes
- */
-router.get('/stats/overview', async (req, res) => {
-  try {
-    const cacheKey = 'stats:overview';
-    const cached = getCached(cacheKey);
-    if (cached) {
-      return res.json(cached);
-    }
-
-    const db = req.app.locals.db;
-    const usersCollection = db.collection('users');
-
-    // Calculate date boundaries
-    const now = new Date();
-    const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
-    const weekStart = new Date(todayStart);
-    weekStart.setDate(weekStart.getDate() - 7);
-    const monthStart = new Date(todayStart);
-    monthStart.setDate(monthStart.getDate() - 30);
-
-    // Execute all queries in parallel
-    const [
-      totalUsers,
-      activeUsers,
-      disabledUsers,
-      newUsersToday,
-      newUsersThisWeek,
-      newUsersThisMonth,
-      loginStats
-    ] = await Promise.all([
-      usersCollection.countDocuments({}),
-      usersCollection.countDocuments({ isActive: true }),
-      usersCollection.countDocuments({ isActive: false }),
-      usersCollection.countDocuments({ createdAt: { $gte: todayStart } }),
-      usersCollection.countDocuments({ createdAt: { $gte: weekStart } }),
-      usersCollection.countDocuments({ createdAt: { $gte: monthStart } }),
-      usersCollection.aggregate([
-        { $group: { _id: null, totalLogins: { $sum: '$loginCount' } } }
-      ]).toArray()
-    ]);
-
-    const totalLogins = loginStats.length > 0 ? loginStats[0].totalLogins : 0;
-    const avgLoginsPerUser = totalUsers > 0
-      ? Math.round((totalLogins / totalUsers) * 10) / 10
-      : 0;
-
-    const result = {
-      totalUsers,
-      activeUsers,
-      disabledUsers,
-      newUsersToday,
-      newUsersThisWeek,
-      newUsersThisMonth,
-      totalLogins,
-      avgLoginsPerUser
-    };
-
-    setCache(cacheKey, result);
-    res.json(result);
-  } catch (error) {
-    console.error('Error fetching stats overview:', error);
-    res.status(500).json({ error: 'Failed to fetch statistics overview' });
-  }
-});
-
-/**
- * GET /api/admin/stats/active-users
- * Returns time series data for active users
- * Query params: period ('daily' | 'weekly' | 'monthly'), days (default: 30)
- * Cached for 5 minutes (different cache key per query params)
- */
-router.get('/stats/active-users', async (req, res) => {
-  try {
-    let { period, days } = req.query;
-
-    // Validate and default period
-    const validPeriods = ['daily', 'weekly', 'monthly'];
-    if (!period || !validPeriods.includes(period)) {
-      period = 'daily';
-    }
-
-    // Validate and default days
-    days = parseInt(days) || 30;
-    days = Math.max(1, Math.min(365, days));
-
-    const cacheKey = `stats:active-users:${period}:${days}`;
-    const cached = getCached(cacheKey);
-    if (cached) {
-      return res.json(cached);
-    }
-
-    const db = req.app.locals.db;
-    const activityCollection = db.collection(ACTIVITY_COLLECTION);
-
-    // Calculate the start date based on period and days
-    const now = new Date();
-    const startDate = new Date(now);
-
-    // Determine date grouping based on period
-    let dateFormat;
-    let dataPoints;
-
-    switch (period) {
-      case 'weekly':
-        // Each data point represents a week
-        startDate.setDate(startDate.getDate() - (days * 7));
-        dataPoints = days;
-        dateFormat = {
-          year: { $year: '$timestamp' },
-          week: { $isoWeek: '$timestamp' }
-        };
-        break;
-      case 'monthly':
-        // Each data point represents a month
-        startDate.setMonth(startDate.getMonth() - days);
-        dataPoints = days;
-        dateFormat = {
-          year: { $year: '$timestamp' },
-          month: { $month: '$timestamp' }
-        };
-        break;
-      case 'daily':
-      default:
-        // Each data point represents a day
-        startDate.setDate(startDate.getDate() - days);
-        dataPoints = days;
-        dateFormat = {
-          year: { $year: '$timestamp' },
-          month: { $month: '$timestamp' },
-          day: { $dayOfMonth: '$timestamp' }
-        };
-        break;
-    }
-
-    // Aggregate unique users per period
-    const pipeline = [
-      {
-        $match: {
-          timestamp: { $gte: startDate }
-        }
-      },
-      {
-        $group: {
-          _id: dateFormat,
-          users: { $addToSet: '$userId' }
-        }
-      },
-      {
-        $project: {
-          _id: 1,
-          count: { $size: '$users' }
-        }
-      },
-      {
-        $sort: { '_id.year': -1, '_id.month': -1, '_id.day': -1, '_id.week': -1 }
-      },
-      {
-        $limit: dataPoints
-      }
-    ];
-
-    const rawData = await activityCollection.aggregate(pipeline).toArray();
-
-    // Format the data into consistent date strings
-    const data = rawData.map(item => {
-      let date;
-      if (period === 'daily') {
-        date = `${item._id.year}-${String(item._id.month).padStart(2, '0')}-${String(item._id.day).padStart(2, '0')}`;
-      } else if (period === 'weekly') {
-        // Format as year-Wweek
-        date = `${item._id.year}-W${String(item._id.week).padStart(2, '0')}`;
-      } else if (period === 'monthly') {
-        date = `${item._id.year}-${String(item._id.month).padStart(2, '0')}`;
-      }
-      return { date, count: item.count };
-    });
-
-    // Sort by date descending (most recent first)
-    data.sort((a, b) => b.date.localeCompare(a.date));
-
-    const result = { period, data };
-    setCache(cacheKey, result);
-    res.json(result);
-  } catch (error) {
-    console.error('Error fetching active users stats:', error);
-    res.status(500).json({ error: 'Failed to fetch active users statistics' });
-  }
-});
-
-/**
- * GET /api/admin/stats/actions
- * Returns action breakdown with counts
- * Query params: startDate, endDate (optional, ISO date strings)
- * Results sorted by count descending
- * Cached for 5 minutes
- */
-router.get('/stats/actions', async (req, res) => {
-  try {
-    const { startDate, endDate } = req.query;
-
-    // Build cache key including date filters
-    const cacheKey = `stats:actions:${startDate || 'all'}:${endDate || 'all'}`;
-    const cached = getCached(cacheKey);
-    if (cached) {
-      return res.json(cached);
-    }
-
-    const db = req.app.locals.db;
-    const activityCollection = db.collection(ACTIVITY_COLLECTION);
-
-    // Build match filter
-    const matchFilter = {};
-    if (startDate || endDate) {
-      matchFilter.timestamp = {};
-      if (startDate && isValidDateString(startDate)) {
-        matchFilter.timestamp.$gte = new Date(startDate);
-      }
-      if (endDate && isValidDateString(endDate)) {
-        matchFilter.timestamp.$lte = new Date(endDate);
-      }
-      // Remove timestamp filter if it's empty
-      if (Object.keys(matchFilter.timestamp).length === 0) {
-        delete matchFilter.timestamp;
-      }
-    }
-
-    // Aggregate action counts
-    const pipeline = [
-      ...(Object.keys(matchFilter).length > 0 ? [{ $match: matchFilter }] : []),
-      {
-        $group: {
-          _id: '$action',
-          count: { $sum: 1 }
-        }
-      },
-      {
-        $project: {
-          _id: 0,
-          action: '$_id',
-          count: 1
-        }
-      },
-      {
-        $sort: { count: -1 }
-      }
-    ];
-
-    const actions = await activityCollection.aggregate(pipeline).toArray();
-
-    const result = { actions };
-    setCache(cacheKey, result);
-    res.json(result);
-  } catch (error) {
-    console.error('Error fetching action stats:', error);
-    res.status(500).json({ error: 'Failed to fetch action statistics' });
-  }
-});
-
-/**
- * GET /api/admin/stats/peak-times
- * Returns hourly (0-23) and daily (Sunday-Saturday) activity averages
- * Based on last 30 days of data
- * Cached for 5 minutes
- */
-router.get('/stats/peak-times', async (req, res) => {
-  try {
-    const cacheKey = 'stats:peak-times';
-    const cached = getCached(cacheKey);
-    if (cached) {
-      return res.json(cached);
-    }
-
-    const db = req.app.locals.db;
-    const activityCollection = db.collection(ACTIVITY_COLLECTION);
-
-    // Calculate 30 days ago
-    const thirtyDaysAgo = new Date();
-    thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
-
-    // Aggregate hourly counts
-    const hourlyPipeline = [
-      {
-        $match: {
-          timestamp: { $gte: thirtyDaysAgo }
-        }
-      },
-      {
-        $group: {
-          _id: { $hour: '$timestamp' },
-          totalActivity: { $sum: 1 },
-          uniqueDays: { $addToSet: {
-            $dateToString: { format: '%Y-%m-%d', date: '$timestamp' }
-          }}
-        }
-      },
-      {
-        $project: {
-          hour: '$_id',
-          avgActivity: {
-            $round: [
-              { $divide: ['$totalActivity', { $size: '$uniqueDays' }] },
-              0
-            ]
-          }
-        }
-      },
-      {
-        $sort: { hour: 1 }
-      }
-    ];
-
-    // Aggregate daily counts (by day of week)
-    const dailyPipeline = [
-      {
-        $match: {
-          timestamp: { $gte: thirtyDaysAgo }
-        }
-      },
-      {
-        $group: {
-          _id: { $dayOfWeek: '$timestamp' }, // 1 = Sunday, 7 = Saturday
-          totalActivity: { $sum: 1 },
-          uniqueWeeks: { $addToSet: {
-            $isoWeek: '$timestamp'
-          }}
-        }
-      },
-      {
-        $project: {
-          dayNum: '$_id',
-          avgActivity: {
-            $round: [
-              { $divide: ['$totalActivity', { $max: [{ $size: '$uniqueWeeks' }, 1] }] },
-              0
-            ]
-          }
-        }
-      },
-      {
-        $sort: { dayNum: 1 }
-      }
-    ];
-
-    const [hourlyRaw, dailyRaw] = await Promise.all([
-      activityCollection.aggregate(hourlyPipeline).toArray(),
-      activityCollection.aggregate(dailyPipeline).toArray()
-    ]);
-
-    // Day of week names (MongoDB uses 1=Sunday, 2=Monday, etc.)
-    const dayNames = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
-
-    // Build complete hourly array (0-23)
-    const hourlyMap = new Map(hourlyRaw.map(h => [h.hour, h.avgActivity]));
-    const hourly = [];
-    for (let hour = 0; hour < 24; hour++) {
-      hourly.push({
-        hour,
-        avgActivity: hourlyMap.get(hour) || 0
-      });
-    }
-
-    // Build complete daily array (Sunday-Saturday)
-    const dailyMap = new Map(dailyRaw.map(d => [d.dayNum, d.avgActivity]));
-    const daily = [];
-    for (let i = 1; i <= 7; i++) {
-      daily.push({
-        day: dayNames[i - 1],
-        avgActivity: dailyMap.get(i) || 0
-      });
-    }
-
-    const result = { hourly, daily };
-    setCache(cacheKey, result);
-    res.json(result);
-  } catch (error) {
-    console.error('Error fetching peak times stats:', error);
-    res.status(500).json({ error: 'Failed to fetch peak times statistics' });
-  }
-});
-
-// ============================================================
-// Settings Endpoints (Phase 4)
-// ============================================================
-
-const SETTINGS_COLLECTION = 'settings';
-const SETTINGS_DOC_ID = 'admin_settings';
-const DEFAULT_RETENTION_DAYS = 90;
-const DEFAULT_LOG_LEVEL = 'all';
-const MIN_RETENTION_DAYS = 30;
-const MAX_RETENTION_DAYS = 365;
-
-/**
- * Get settings document or return defaults
- * @param {Db} db - MongoDB database instance
- * @returns {Promise<Object>} Settings object
- */
-async function getSettings(db) {
-  const settings = await db.collection(SETTINGS_COLLECTION).findOne({ _id: SETTINGS_DOC_ID });
-  if (!settings) {
-    return {
-      activityRetentionDays: DEFAULT_RETENTION_DAYS,
-      activityLogLevel: DEFAULT_LOG_LEVEL
-    };
-  }
-  return {
-    activityRetentionDays: settings.activityRetentionDays ?? DEFAULT_RETENTION_DAYS,
-    activityLogLevel: settings.activityLogLevel ?? DEFAULT_LOG_LEVEL
-  };
-}
-
-/**
- * Update TTL index on user_activity collection
- * @param {Db} db - MongoDB database instance
- * @param {number} days - Retention period in days
- */
-async function updateTTLIndex(db, days) {
-  const collection = db.collection(ACTIVITY_COLLECTION);
-  const expireAfterSeconds = days * 24 * 60 * 60;
-
-  try {
-    // Try to drop existing TTL index
-    await collection.dropIndex('timestamp_ttl');
-  } catch (err) {
-    // Index may not exist, which is fine
-    if (err.codeName !== 'IndexNotFound') {
-      console.warn('Note: timestamp_ttl index not found, will create new one');
-    }
-  }
-
-  // Create new TTL index with updated expiry
-  await collection.createIndex(
-    { timestamp: 1 },
-    {
-      name: 'timestamp_ttl',
-      expireAfterSeconds: expireAfterSeconds
-    }
-  );
-}
-
-/**
- * GET /api/admin/settings
- * Returns current admin settings
- */
-router.get('/settings', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-    const settings = await getSettings(db);
-    res.json(settings);
-  } catch (error) {
-    console.error('Error fetching settings:', error);
-    res.status(500).json({ error: 'Failed to fetch settings' });
-  }
-});
-
-/**
- * PATCH /api/admin/settings
- * Update admin settings
- */
-router.patch('/settings', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-    const { activityRetentionDays, activityLogLevel } = req.body;
-
-    // Validate retention period if provided
-    if (activityRetentionDays !== undefined) {
-      // Check if it's a valid number
-      if (typeof activityRetentionDays !== 'number' || !Number.isInteger(activityRetentionDays)) {
-        return res.status(400).json({ error: 'activityRetentionDays must be an integer' });
-      }
-
-      // Check range
-      if (activityRetentionDays < MIN_RETENTION_DAYS || activityRetentionDays > MAX_RETENTION_DAYS) {
-        return res.status(400).json({
-          error: `activityRetentionDays must be between ${MIN_RETENTION_DAYS} and ${MAX_RETENTION_DAYS} days`
-        });
-      }
-    }
-
-    // Validate log level if provided
-    const validLogLevels = ['all', 'navigation', 'none'];
-    if (activityLogLevel !== undefined && !validLogLevels.includes(activityLogLevel)) {
-      return res.status(400).json({
-        error: `activityLogLevel must be one of: ${validLogLevels.join(', ')}`
-      });
-    }
-
-    // Build update object
-    const updateFields = {
-      updatedAt: new Date(),
-      updatedBy: req.user._id
-    };
-
-    if (activityRetentionDays !== undefined) {
-      updateFields.activityRetentionDays = activityRetentionDays;
-    }
-    if (activityLogLevel !== undefined) {
-      updateFields.activityLogLevel = activityLogLevel;
-    }
-
-    // Build $setOnInsert only for fields NOT in $set
-    const setOnInsertFields = {};
-    if (activityRetentionDays === undefined) {
-      setOnInsertFields.activityRetentionDays = DEFAULT_RETENTION_DAYS;
-    }
-    if (activityLogLevel === undefined) {
-      setOnInsertFields.activityLogLevel = DEFAULT_LOG_LEVEL;
-    }
-
-    // Upsert settings document
-    const updateDoc = { $set: updateFields };
-    if (Object.keys(setOnInsertFields).length > 0) {
-      updateDoc.$setOnInsert = setOnInsertFields;
-    }
-
-    await db.collection(SETTINGS_COLLECTION).updateOne(
-      { _id: SETTINGS_DOC_ID },
-      updateDoc,
-      { upsert: true }
-    );
-
-    // Update TTL index if retention period changed
-    if (activityRetentionDays !== undefined) {
-      await updateTTLIndex(db, activityRetentionDays);
-    }
-
-    // Log admin action
-    await logActivity(db, {
-      userId: req.user._id.toString(),
-      action: 'ADMIN_UPDATE_SETTINGS',
-      metadata: {
-        ...(activityRetentionDays !== undefined && { activityRetentionDays }),
-        ...(activityLogLevel !== undefined && { activityLogLevel })
-      }
-    });
-
-    // Fetch updated settings
-    const updatedSettings = await getSettings(db);
-
-    res.json({
-      data: {
-        settings: updatedSettings,
-        message: 'Settings updated successfully'
-      }
-    });
-  } catch (error) {
-    console.error('Error updating settings:', error);
-    res.status(500).json({ error: 'Failed to update settings' });
-  }
-});
-
-// ============================================================
-// Scraper Endpoints
-// ============================================================
-
-const crypto = require('crypto');
-const {
-  isScraperRunning,
-  acquireLock,
-  releaseLock,
-  getCurrentJobId,
-  getScheduleExpression,
-  getNextScheduledRun
-} = require('../jobs/scraperJob');
-const { runScrape } = require('../services/scraperService');
-const { createLogger } = require('../services/scraperLogger');
-const scraperConfig = require('../config/scraper');
-
-// Logger for scraper admin routes
-const scraperRouteLogger = createLogger('scraper-admin');
-
-// Rate limiting for manual scrape trigger (per-user, in-memory)
-const RATE_LIMIT_MAX_REQUESTS = 5;
-const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
-const rateLimitStore = new Map();
-
-/**
- * Check rate limit for a given user ID.
- * Returns an object indicating whether the request is allowed.
- *
- * @param {string} userId - The user ID to check
- * @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
- */
-function checkRateLimit(userId) {
-  const now = Date.now();
-  const userKey = userId.toString();
-
-  if (!rateLimitStore.has(userKey)) {
-    rateLimitStore.set(userKey, []);
-  }
-
-  const timestamps = rateLimitStore.get(userKey);
-
-  // Remove timestamps outside the current window
-  const windowStart = now - RATE_LIMIT_WINDOW_MS;
-  const validTimestamps = timestamps.filter(ts => ts > windowStart);
-  rateLimitStore.set(userKey, validTimestamps);
-
-  if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
-    // Calculate when the oldest request in the window will expire
-    const oldestTimestamp = validTimestamps[0];
-    const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
-    const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
-    return { allowed: false, retryAfterSeconds };
-  }
-
-  // Record this request
-  validTimestamps.push(now);
-  return { allowed: true, retryAfterSeconds: null };
-}
-
-/**
- * Reset the rate limiter (for testing)
- */
-function resetRateLimiter() {
-  rateLimitStore.clear();
-}
-
-/**
- * POST /api/admin/scraper/run
- * Trigger a manual scrape
- *
- * Request body (optional):
- *   - dryRun: boolean - Skip database writes for safe testing
- *   - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
- */
-router.post('/scraper/run', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-    const { dryRun = false, htmlContent = null } = req.body || {};
-
-    // Check rate limit (per-user, before mutex check)
-    const rateLimitResult = checkRateLimit(req.user._id);
-    if (!rateLimitResult.allowed) {
-      res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
-      return res.status(429).json({
-        error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
-      });
-    }
-
-    // Check if scraper is already running
-    if (isScraperRunning()) {
-      return res.status(409).json({ error: 'Scrape already in progress' });
-    }
-
-    // Generate job ID and acquire lock
-    const jobId = crypto.randomUUID();
-    acquireLock(jobId);
-
-    // Log admin action
-    await logActivity(db, {
-      userId: req.user._id.toString(),
-      action: 'ADMIN_TRIGGER_SCRAPE',
-      metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
-    });
-
-    // Start scrape asynchronously (do not await - return 202 immediately)
-    runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
-      .catch((error) => {
-        scraperRouteLogger.error('Async scrape failed', {
-          errorType: error.name,
-          errorMessage: error.message
-        });
-      })
-      .finally(() => releaseLock());
-
-    // Return immediately with job ID
-    res.status(202).json({
-      data: {
-        jobId,
-        status: 'started',
-        dryRun,
-        message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
-      }
-    });
-
-  } catch (error) {
-    scraperRouteLogger.error('Error triggering scrape', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    res.status(500).json({ error: 'Failed to start scrape job' });
-  }
-});
-
-/**
- * GET /api/admin/scraper/status
- * Get current scraper status including running state, last run details,
- * next scheduled run, and schedule expression.
- */
-router.get('/scraper/status', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-
-    // Get last run from history
-    const lastRunDoc = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
-      .findOne({}, { sort: { startedAt: -1 } });
-
-    const lastRun = lastRunDoc ? {
-      jobId: lastRunDoc.jobId,
-      timestamp: lastRunDoc.startedAt,
-      status: lastRunDoc.status,
-      duration: lastRunDoc.duration,
-      trigger: lastRunDoc.trigger,
-      unitsProcessed: lastRunDoc.unitsProcessed,
-      pricesInserted: lastRunDoc.pricesInserted,
-      errors: lastRunDoc.errors?.length > 0 ? lastRunDoc.errors : null
-    } : null;
-
-    res.json({
-      data: {
-        currentStatus: isScraperRunning() ? 'running' : 'idle',
-        runningJobId: isScraperRunning() ? getCurrentJobId() : null,
-        lastRun,
-        nextScheduledRun: getNextScheduledRun(),
-        schedule: getScheduleExpression()
-      }
-    });
-
-  } catch (error) {
-    scraperRouteLogger.error('Error fetching scraper status', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    res.status(503).json({ error: 'Service temporarily unavailable' });
-  }
-});
-
-/**
- * GET /api/admin/scraper/history
- * Get scraper run history with pagination
- *
- * Query params:
- *   - limit: Number of records (1-100, default 30)
- *   - offset: Number of records to skip (default 0)
- */
-router.get('/scraper/history', async (req, res) => {
-  try {
-    const db = req.app.locals.db;
-
-    // Parse and validate pagination parameters
-    let limit = parseInt(req.query.limit);
-    let offset = parseInt(req.query.offset);
-
-    // Validate limit
-    if (req.query.limit !== undefined) {
-      if (isNaN(limit) || limit < 1) {
-        return res.status(400).json({ error: 'Limit must be between 1 and 100' });
-      }
-      limit = Math.min(limit, 100);
-    } else {
-      limit = 30;
-    }
-
-    // Validate offset
-    if (req.query.offset !== undefined) {
-      if (isNaN(offset) || offset < 0) {
-        return res.status(400).json({ error: 'Invalid offset parameter' });
-      }
-    } else {
-      offset = 0;
-    }
-
-    const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
-      .find({})
-      .sort({ startedAt: -1 })
-      .skip(offset)
-      .limit(limit)
-      .toArray();
-
-    res.json({ data: history });
-
-  } catch (error) {
-    scraperRouteLogger.error('Error fetching scraper history', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    res.status(503).json({ error: 'Service temporarily unavailable' });
-  }
-});
-
 module.exports = router;
-module.exports.clearStatsCache = clearStatsCache;
-module.exports.resetRateLimiter = resetRateLimiter;

server.js

@@ -11,9 +11,6 @@ const activityRoutes = require('./routes/activity');
 const adminRoutes = require('./routes/admin');
 const { createIndexes } = require('./models/user');
 const { createActivityIndexes } = require('./services/activityLogger');
-const { createScraperIndexes } = require('./services/scraperService');
-const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
-const scraperConfig = require('./config/scraper');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
@@ -83,32 +80,14 @@ async function connectToMongoDB() {
     await createIndexes(db);
     await createActivityIndexes(db);
     console.log('✅ Passport configured and indexes created');
-
-    // Initialize scraper (indexes + scheduler) - errors logged but don't crash server
-    try {
-      const { createLogger } = require('./services/scraperLogger');
-      const scraperLogger = createLogger('server-init');
-      await createScraperIndexes(db, scraperLogger);
-      if (scraperConfig.SCRAPER_ENABLED) {
-        initializeScheduler(db);
-        console.log('✅ Scraper scheduler initialized');
-      } else {
-        console.log('ℹ️ Scraper scheduling disabled');
-      }
-    } catch (error) {
-      console.error('⚠️ Scraper initialization failed (server continues):', error.message);
-    }
   } catch (error) {
     console.error('❌ Failed to connect to MongoDB:', error);
     process.exit(1);
   }
 }
 
-// Register scraper signal handlers for graceful shutdown
+// Helper function to get today's date in UTC
-registerSignalHandlers();
+const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
-
-// Helper function to get today's date in configured timezone
-const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
 
 // Cache for the most recent date with data
 let cachedLatestDate = null;
@@ -143,7 +122,7 @@ const getLatestDateWithData = async () => {
   }
 
   // Fallback to UTC today if no data found
-  return getTodayDate();
+  return getTodayDateUTC();
 };
 
 // Helper function to get yesterday relative to the latest date with data
@@ -168,8 +147,8 @@ app.use('/auth', authRoutes);
 // Mount activity routes
 app.use('/activity', activityRoutes);
 
-// Mount admin routes (Traefik strips /api prefix, so /api/admin becomes /admin)
+// Mount admin routes
-app.use('/admin', adminRoutes);
+app.use('/api/admin', adminRoutes);
 
 // Health check endpoint
 app.get('/health', (req, res) => {
@@ -312,7 +291,6 @@ app.get('/price-history', requireAuth, async (req, res) => {
 app.get('/available-units', requireAuth, async (req, res) => {
   try {
     const today = await getLatestDateWithData();
-    const yesterday = getYesterdayDate(today);
     
     // Get units that have prices today (excluding furnished units)
     const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
@@ -337,17 +315,11 @@ app.get('/available-units', requireAuth, async (req, res) => {
       },
       {
         $addFields: {
-          currentPriceDoc: {
+          currentPrice: {
             $arrayElemAt: [
               { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", today] } } },
               0
             ]
-          },
-          yesterdayPriceDoc: {
-            $arrayElemAt: [
-              { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", yesterday] } } },
-              0
-            ]
           }
         }
       },
@@ -436,17 +408,6 @@ app.get('/available-units', requireAuth, async (req, res) => {
           }
         }
       },
-      {
-        $addFields: {
-          priceChangeToday: {
-            $cond: {
-              if: { $and: [{ $ne: ["$currentPriceDoc.price", null] }, { $ne: ["$yesterdayPriceDoc.price", null] }] },
-              then: { $subtract: ["$currentPriceDoc.price", "$yesterdayPriceDoc.price"] },
-              else: null
-            }
-          }
-        }
-      },
       {
         $project: {
           unitCode: "$unit_code",
@@ -457,9 +418,7 @@ app.get('/available-units', requireAuth, async (req, res) => {
           community: "$community",
           available: "$available",
           soonest: "$soonest",
-          currentPrice: "$currentPriceDoc.price",
+          currentPrice: "$currentPrice.price",
-          yesterdayPrice: "$yesterdayPriceDoc.price",
-          priceChangeToday: { $ifNull: ["$priceChangeToday", null] },
           minPrice: "$priceStats.minPrice",
           maxPrice: "$priceStats.maxPrice",
           daysAvailable: {
@@ -641,10 +600,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
     ]).toArray();
 
     // 4. Historical Comparison - current vs 30/90/365 day averages
-    const tz = scraperConfig.SCRAPER_TIMEZONE;
+    const now = new Date();
-    const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
-    const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
-    const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
 
     const [currentAvg, avg30, avg90, avg365] = await Promise.all([
       db.collection(PRICES_COLLECTION).aggregate([

services/activityLogger.js

@@ -69,68 +69,37 @@ function shouldLog(action) {
   return allowedActions.includes(action);
 }
 
-/**
- * Check if an action is an admin action (always logged regardless of log level)
- * @param {string} action - Action to check
- * @returns {boolean} True if admin action
- */
-function isAdminAction(action) {
-  return action && action.startsWith('ADMIN_');
-}
-
 /**
  * Log a user activity to the database
- * Supports two calling conventions:
- * 1. logActivity(db, userId, action, metadata, req) - positional parameters
- * 2. logActivity(db, { userId, action, metadata }) - object parameter for admin actions
- *
  * @param {Db} db - MongoDB database instance
- * @param {string|Object} userIdOrOptions - User ID string or options object
+ * @param {string} userId - User ID (will be converted to ObjectId)
- * @param {string} [action] - Action type (use ACTIONS constants)
+ * @param {string} action - Action type (use ACTIONS constants)
- * @param {Object} [metadata] - Additional action-specific data
+ * @param {Object} metadata - Additional action-specific data
- * @param {Object} [req] - Express request object (for IP and user agent)
+ * @param {Object} req - Express request object (for IP and user agent)
  * @returns {Promise<void>}
  */
-async function logActivity(db, userIdOrOptions, action, metadata = {}, req = null) {
+async function logActivity(db, userId, action, metadata = {}, req = null) {
-  let userId;
+  // Only log if this action type is enabled at current log level
-  let actualAction;
+  if (!shouldLog(action)) {
-  let actualMetadata;
-  let actualReq;
-
-  // Support object-based call for admin actions
-  if (typeof userIdOrOptions === 'object' && userIdOrOptions !== null) {
-    userId = userIdOrOptions.userId;
-    actualAction = userIdOrOptions.action;
-    actualMetadata = userIdOrOptions.metadata || {};
-    actualReq = userIdOrOptions.req || null;
-  } else {
-    userId = userIdOrOptions;
-    actualAction = action;
-    actualMetadata = metadata;
-    actualReq = req;
-  }
-
-  // Admin actions bypass log level filtering
-  if (!isAdminAction(actualAction) && !shouldLog(actualAction)) {
     return;
   }
 
   try {
     // Extract IP address (handle proxy forwarding)
     let ip = null;
-    if (actualReq) {
+    if (req) {
-      ip = actualReq.ip || actualReq.headers?.['x-forwarded-for']?.split(',')[0] || null;
+      ip = req.ip || req.headers?.['x-forwarded-for']?.split(',')[0] || null;
     }
 
     // Extract user agent
-    const userAgent = actualReq?.headers?.['user-agent'] || null;
+    const userAgent = req?.headers?.['user-agent'] || null;
 
     // Create activity document
     const activityDoc = {
       userId: new ObjectId(userId),
-      action: actualAction,
+      action: action,
-      metadata: actualMetadata || {},
+      metadata: metadata || {},
-      page: actualMetadata?.page || null,
+      page: metadata?.page || null,
       timestamp: new Date(),
       userAgent: userAgent,
       ip: ip

services/scraperLogger.js

@@ -1,164 +0,0 @@
-/**
- * Scraper-specific structured JSON logger
- * Produces one JSON object per line to stdout
- */
-
-const LEVELS = {
-  info: 'info',
-  warn: 'warn',
-  error: 'error'
-};
-
-/**
- * Create a logger instance scoped to a job ID
- * @param {string} jobId - Job identifier for correlation
- * @returns {Object} Logger object with info, warn, error methods
- */
-function createLogger(jobId) {
-  /**
-   * Internal log function
-   * @param {string} level - Log level
-   * @param {string} message - Log message
-   * @param {Object} context - Additional context data
-   */
-  const log = (level, message, context = {}) => {
-    const entry = {
-      timestamp: new Date().toISOString(),
-      level,
-      message: truncateMessage(message),
-      jobId,
-      context: sanitizeContext(context)
-    };
-
-    // Output as single-line JSON
-    console.log(JSON.stringify(entry));
-  };
-
-  return {
-    info: (message, context) => log(LEVELS.info, message, context),
-    warn: (message, context) => log(LEVELS.warn, message, context),
-    error: (message, context) => log(LEVELS.error, message, context)
-  };
-}
-
-/**
- * Truncate message to prevent log bloat
- * @param {string} message - Message to truncate
- * @param {number} maxLength - Maximum length (default 1000)
- * @returns {string} Truncated message
- */
-function truncateMessage(message, maxLength = 1000) {
-  if (message === null || message === undefined) {
-    return '';
-  }
-  const str = String(message);
-  if (str.length <= maxLength) {
-    return str;
-  }
-  return str.substring(0, maxLength) + '... [truncated]';
-}
-
-/**
- * Recursively process an object to handle Buffers, circular references, and sensitive data
- * @param {*} obj - Object to process
- * @param {WeakSet} seen - Set of seen objects for circular reference detection
- * @returns {*} Processed value
- */
-function processValue(obj, seen = new WeakSet()) {
-  // Handle null/undefined
-  if (obj === null) {
-    return null;
-  }
-  if (obj === undefined) {
-    return null;
-  }
-
-  // Handle Buffer BEFORE checking for object (Buffer is an object)
-  if (Buffer.isBuffer(obj)) {
-    return `[Buffer: ${obj.length} bytes]`;
-  }
-
-  // Handle strings - check for MongoDB connection strings
-  if (typeof obj === 'string') {
-    if (/mongodb(\+srv)?:\/\//.test(obj)) {
-      return redactConnectionString(obj);
-    }
-    return obj;
-  }
-
-  // Handle primitives
-  if (typeof obj !== 'object') {
-    return obj;
-  }
-
-  // Handle circular references
-  if (seen.has(obj)) {
-    return '[Circular]';
-  }
-  seen.add(obj);
-
-  // Handle arrays
-  if (Array.isArray(obj)) {
-    return obj.map(item => processValue(item, seen));
-  }
-
-  // Handle plain objects
-  const result = {};
-  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
-
-  for (const key of Object.keys(obj)) {
-    // Check for sensitive keys
-    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
-      result[key] = '[REDACTED]';
-    } else {
-      result[key] = processValue(obj[key], seen);
-    }
-  }
-
-  return result;
-}
-
-/**
- * Sanitize context object for safe logging
- * - Remove circular references
- * - Redact sensitive data
- * - Handle special types (Buffer, undefined)
- * @param {Object} context - Context object
- * @returns {Object} Sanitized context
- */
-function sanitizeContext(context) {
-  if (!context || typeof context !== 'object') {
-    return {};
-  }
-
-  try {
-    return processValue(context);
-  } catch (error) {
-    // If sanitization fails, return empty context
-    return { sanitizationError: 'Failed to sanitize context' };
-  }
-}
-
-/**
- * Redact credentials from MongoDB connection string
- * @param {string} uri - Connection string
- * @returns {string} Redacted string
- */
-function redactConnectionString(uri) {
-  try {
-    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
-    return uri.replace(
-      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
-      'mongodb$1://[user]:[REDACTED]@'
-    );
-  } catch {
-    return '[REDACTED CONNECTION STRING]';
-  }
-}
-
-module.exports = {
-  createLogger,
-  LEVELS,
-  // Export internal functions for testing
-  redactConnectionString
-};

services/scraperService.js

@@ -1,920 +0,0 @@
-/**
- * Scraper Service
- *
- * Core scraper logic including HTTP fetching, HTML parsing,
- * data transformation, and database operations.
- */
-
-const axios = require('axios');
-const cheerio = require('cheerio');
-const crypto = require('crypto');
-const config = require('../config/scraper');
-const { createLogger } = require('./scraperLogger');
-
-/**
- * Sleep utility for retry delays
- * @param {number} ms - Milliseconds to sleep
- * @returns {Promise<void>}
- */
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/**
- * Determine if an error is retryable
- * @param {Error} error - Axios error
- * @returns {boolean} True if should retry
- */
-function isRetryableError(error) {
-  // Network errors (timeout, DNS, connection) don't have a response property
-  if (!error.response) {
-    return true;
-  }
-
-  const status = error.response.status;
-
-  // 5xx server errors are retryable
-  if (status >= 500) {
-    return true;
-  }
-
-  // 429 Too Many Requests - do not retry immediately
-  if (status === 429) {
-    return false;
-  }
-
-  // 4xx client errors - do not retry
-  return false;
-}
-
-/**
- * Fetch page HTML with retry logic
- * @param {string} url - Target URL
- * @param {Object} logger - Logger instance
- * @returns {Promise<string>} HTML content
- * @throws {Error} After all retries exhausted
- */
-async function fetchPage(url, logger) {
-  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
-  let lastError;
-
-  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
-    try {
-      logger.info('Fetching page', { url, attempt });
-
-      const response = await axios.get(url, {
-        timeout,
-        headers: {
-          'User-Agent': config.USER_AGENT
-        },
-        maxRedirects: 5,
-        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
-      });
-
-      logger.info('Page fetched successfully', {
-        status: response.status,
-        contentLength: response.data.length
-      });
-
-      return response.data;
-
-    } catch (error) {
-      lastError = error;
-
-      // Determine if error is retryable
-      const isRetryable = isRetryableError(error);
-
-      logger.warn('Fetch attempt failed', {
-        attempt,
-        errorType: error.name,
-        errorMessage: error.message,
-        statusCode: error.response?.status,
-        isRetryable
-      });
-
-      // Don't retry non-retryable errors (4xx)
-      if (!isRetryable) {
-        throw error;
-      }
-
-      // Don't wait after last attempt
-      if (attempt <= maxRetries) {
-        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
-        logger.info('Waiting before retry', { delay });
-        await sleep(delay);
-      }
-    }
-  }
-
-  throw lastError;
-}
-
-/**
- * Parse unit data from HTML using cheerio
- * @param {string} html - HTML content
- * @param {Object} logger - Logger instance
- * @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
- */
-function parseUnits(html, logger) {
-  const $ = cheerio.load(html);
-  const units = [];
-
-  // Find the main container
-  const container = $('section.spaces__tab-unit');
-
-  if (container.length === 0) {
-    logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
-    return [];
-  }
-
-  // Extract each article element
-  container.find('article').each((index, article) => {
-    const $article = $(article);
-
-    const unit = {
-      // Core identifiers
-      id: $article.attr('data-spaces-id'),
-      unit_code: $article.attr('data-spaces-unit'),
-      unit_id: $article.attr('data-spaces-unit-id'),
-
-      // Physical attributes
-      floor: $article.attr('data-spaces-unit-floor'),
-      area: $article.attr('data-spaces-sort-area'),
-      bed_count: $article.attr('data-spaces-sort-bed'),
-      bath_count: $article.attr('data-spaces-sort-bath'),
-
-      // Pricing
-      price: $article.attr('data-spaces-sort-price'),
-
-      // Availability
-      available: $article.attr('data-spaces-available'),
-      unavailable: $article.attr('data-spaces-unavailable'),
-      soonest: $article.attr('data-spaces-soonest'),
-      date_available: $article.attr('data-spaces-sort-date'),
-
-      // Plan information
-      plan_id: $article.attr('data-spaces-plan-id'),
-      plan_name: $article.attr('data-spaces-sort-plan-name'),
-
-      // Property information
-      obj_type: $article.attr('data-spaces-obj'),
-      community: $article.attr('data-spaces-community'),
-      asset: $article.attr('data-spaces-asset'),
-
-      // URLs
-      href: $article.attr('data-spaces-href'),
-      inventory_href: $article.attr('data-spaces-inventory-href'),
-
-      // Specials
-      specials_content: $article.attr('data-spaces-specials-content')
-    };
-
-    // Extract image URL from nested element if present
-    const imgElement = $article.find('img').first();
-    if (imgElement.length > 0) {
-      unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
-    }
-
-    units.push(unit);
-  });
-
-  logger.info('Units parsed', { count: units.length });
-
-  // Deduplicate by unit_code (in case of duplicate articles)
-  const seen = new Set();
-  const deduplicated = units.filter(unit => {
-    if (!unit.unit_code || seen.has(unit.unit_code)) {
-      return false;
-    }
-    seen.add(unit.unit_code);
-    return true;
-  });
-
-  if (deduplicated.length < units.length) {
-    logger.warn('Duplicate units removed', {
-      original: units.length,
-      deduplicated: deduplicated.length
-    });
-  }
-
-  return deduplicated;
-}
-
-// ============================================================
-// Data Type Conversion Helpers
-// ============================================================
-
-/**
- * Parse string to integer, return null for invalid
- * @param {*} value - Value to parse
- * @returns {number|null} Parsed integer or null
- */
-function parseInteger(value) {
-  if (value === null || value === undefined || value === '') {
-    return null;
-  }
-  const parsed = parseInt(value, 10);
-  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
-}
-
-/**
- * Parse string to positive integer, return null for 0 or invalid.
- * Used for fields like area where 0 means "not available".
- * @param {*} value - Value to parse
- * @returns {number|null} Parsed positive integer or null
- */
-function parsePositiveInteger(value) {
-  const parsed = parseInteger(value);
-  return parsed === 0 ? null : parsed;
-}
-
-/**
- * Parse value that could be integer or string identifier.
- * Returns integer if cleanly parseable, otherwise trimmed string.
- * @param {*} value - Value to parse
- * @returns {number|string|null} Parsed integer, trimmed string, or null
- */
-function parseIntegerOrString(value) {
-  if (value === null || value === undefined || value === '') {
-    return null;
-  }
-  const parsed = parseInt(value, 10);
-  // If it parses cleanly to an integer, return integer
-  if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
-    return parsed;
-  }
-  // Otherwise return as trimmed string
-  return String(value).trim();
-}
-
-/**
- * Parse price, stripping non-numeric characters.
- * Handles "$1,234" format and "Call for pricing" text.
- * @param {*} value - Value to parse
- * @returns {number|null} Parsed price or null
- */
-function parsePrice(value) {
-  if (value === null || value === undefined || value === '') {
-    return null;
-  }
-
-  // Check for "Call for pricing" or similar text
-  if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
-    return null;
-  }
-
-  // Strip non-numeric characters except decimal point
-  const cleaned = String(value).replace(/[^0-9.]/g, '');
-  const parsed = parseInt(cleaned, 10);
-
-  if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
-    return null;
-  }
-
-  // Negative prices are invalid
-  if (parsed < 0) {
-    return null;
-  }
-
-  return parsed;
-}
-
-/**
- * Parse boolean from string.
- * Handles "true"/"false"/"1"/"0" and actual boolean values.
- * @param {*} value - Value to parse
- * @returns {boolean|null} Parsed boolean or null
- */
-function parseBoolean(value) {
-  if (value === null || value === undefined || value === '') {
-    return null;
-  }
-  if (typeof value === 'boolean') {
-    return value;
-  }
-  const str = String(value).toLowerCase().trim();
-  if (str === 'true' || str === '1') return true;
-  if (str === 'false' || str === '0') return false;
-  return null;
-}
-
-/**
- * Parse float value.
- * Named parseFloatValue to avoid shadowing the global parseFloat.
- * @param {*} value - Value to parse
- * @returns {number|null} Parsed float or null
- */
-function parseFloatValue(value) {
-  if (value === null || value === undefined || value === '') {
-    return null;
-  }
-  const parsed = Number.parseFloat(value);
-  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
-}
-
-/**
- * Trim string, return null for empty.
- * Converts non-string values to string before trimming.
- * @param {*} value - Value to trim
- * @returns {string|null} Trimmed string or null
- */
-function trimString(value) {
-  if (value === null || value === undefined) {
-    return null;
-  }
-  const trimmed = String(value).trim();
-  return trimmed === '' ? null : trimmed;
-}
-
-// ============================================================
-// Main Data Type Conversion Function
-// ============================================================
-
-/**
- * Convert unit data types from strings to proper types.
- * Applies the appropriate parser to each field based on its expected type.
- * @param {Object} unit - Raw unit object with string values
- * @returns {Object} Unit object with converted types
- */
-function convertDataTypes(unit) {
-  return {
-    // Integer fields
-    id: parseInteger(unit.id),
-    unit_id: parseIntegerOrString(unit.unit_id),
-    floor: parseInteger(unit.floor),
-    area: parsePositiveInteger(unit.area),
-    bed_count: parseInteger(unit.bed_count),
-    plan_id: parseInteger(unit.plan_id),
-    asset: parseInteger(unit.asset),
-    date_available: parseInteger(unit.date_available),
-
-    // Float fields
-    bath_count: parseFloatValue(unit.bath_count),
-
-    // Price field (special handling)
-    price: parsePrice(unit.price),
-
-    // Boolean fields
-    available: parseBoolean(unit.available),
-    unavailable: parseBoolean(unit.unavailable),
-
-    // String fields (trim and preserve)
-    unit_code: trimString(unit.unit_code),
-    plan_name: trimString(unit.plan_name),
-    soonest: trimString(unit.soonest),
-    obj_type: trimString(unit.obj_type),
-    community: trimString(unit.community),
-    href: trimString(unit.href),
-    inventory_href: trimString(unit.inventory_href),
-    image_url: trimString(unit.image_url),
-    specials_content: trimString(unit.specials_content)
-  };
-}
-
-// ============================================================
-// Date Helpers
-// ============================================================
-
-/**
- * Calculate yesterday's date from a given date string.
- * @param {string} dateStr - Date in YYYY-MM-DD format
- * @returns {string} Yesterday's date in YYYY-MM-DD format
- */
-function getYesterday(dateStr) {
-  const date = new Date(dateStr + 'T00:00:00Z');
-  date.setUTCDate(date.getUTCDate() - 1);
-  return date.toISOString().split('T')[0];
-}
-
-// ============================================================
-// Database Operations
-// ============================================================
-
-/**
- * Upsert unit records to database using bulkWrite.
- * Each unit is matched by unit_code as the unique key.
- * Sets last_scraped and data_source on every update.
- * Sets first_seen only on initial insert via $setOnInsert.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Array<Object>} units - Array of unit objects
- * @param {Object} logger - Logger instance
- * @returns {Promise<Object>} Bulk write result
- */
-async function upsertUnits(db, units, logger) {
-  const collection = db.collection(config.COLLECTIONS.UNITS);
-  const now = new Date().toISOString();
-
-  const operations = units.map(unit => ({
-    updateOne: {
-      filter: { unit_code: unit.unit_code },
-      update: {
-        $set: {
-          ...unit,
-          last_scraped: now,
-          data_source: 'web_scraper'
-        },
-        $setOnInsert: {
-          first_seen: now
-        }
-      },
-      upsert: true
-    }
-  }));
-
-  if (operations.length === 0) {
-    logger.warn('No units to upsert');
-    return { modifiedCount: 0, upsertedCount: 0 };
-  }
-
-  try {
-    const result = await collection.bulkWrite(operations, { ordered: false });
-
-    logger.info('Units upserted', {
-      matched: result.matchedCount,
-      modified: result.modifiedCount,
-      upserted: result.upsertedCount
-    });
-
-    return result;
-
-  } catch (error) {
-    logger.error('Failed to upsert units', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    throw error;
-  }
-}
-
-/**
- * Insert price records for today using bulkWrite with upsert.
- * Uses unit_code + date_checked as the composite key to prevent
- * duplicate price records for the same unit on the same day.
- * Re-running on the same day updates existing records (idempotent).
- *
- * @param {Db} db - MongoDB database instance
- * @param {Array<Object>} units - Array of unit objects
- * @param {string} date - Date in YYYY-MM-DD format
- * @param {Object} logger - Logger instance
- * @returns {Promise<Object>} Insert result with insertedCount
- */
-async function insertPrices(db, units, date, logger) {
-  const collection = db.collection(config.COLLECTIONS.PRICES);
-  const now = new Date().toISOString();
-
-  // Filter out units without prices
-  const unitsWithPrices = units.filter(u => u.price !== null);
-
-  const priceRecords = unitsWithPrices.map(unit => ({
-    unit_code: unit.unit_code,
-    date_checked: date,
-    price: unit.price,
-    last_updated: now,
-    data_source: 'web_scraper'
-  }));
-
-  if (priceRecords.length === 0) {
-    logger.warn('No price records to insert');
-    return { insertedCount: 0 };
-  }
-
-  // Use updateOne with upsert to handle re-runs on same day
-  const operations = priceRecords.map(record => ({
-    updateOne: {
-      filter: {
-        unit_code: record.unit_code,
-        date_checked: record.date_checked
-      },
-      update: { $set: record },
-      upsert: true
-    }
-  }));
-
-  try {
-    const result = await collection.bulkWrite(operations, { ordered: false });
-
-    logger.info('Prices inserted', {
-      inserted: result.upsertedCount,
-      updated: result.modifiedCount
-    });
-
-    return { insertedCount: result.upsertedCount + result.modifiedCount };
-
-  } catch (error) {
-    logger.error('Failed to insert prices', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    throw error;
-  }
-}
-
-/**
- * Mark units not in current scrape as stale/unavailable.
- * Uses updateMany to set available: false and marked_stale_date
- * for all units whose unit_code is NOT in the current scrape
- * and that are currently available.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Set<string>} currentUnitCodes - Unit codes from current scrape
- * @param {string} date - Current date in YYYY-MM-DD format
- * @param {Object} logger - Logger instance
- * @returns {Promise<Object>} Update result
- */
-async function markStaleUnits(db, currentUnitCodes, date, logger) {
-  const collection = db.collection(config.COLLECTIONS.UNITS);
-
-  try {
-    const result = await collection.updateMany(
-      {
-        unit_code: { $nin: [...currentUnitCodes] },
-        available: true
-      },
-      {
-        $set: {
-          available: false,
-          marked_stale_date: date
-        }
-      }
-    );
-
-    logger.info('Stale units marked', { count: result.modifiedCount });
-
-    return result;
-
-  } catch (error) {
-    logger.error('Failed to mark stale units', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    throw error;
-  }
-}
-
-/**
- * Update daily summary document.
- * Upserts by date field (YYYY-MM-DD format).
- * Fetches yesterday's summary for comparison metrics.
- * Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
- *
- * @param {Db} db - MongoDB database instance
- * @param {Object} summaryData - Summary data
- * @param {string} summaryData.date - Date in YYYY-MM-DD format
- * @param {Array<string>} summaryData.newUnits - Unit codes added today
- * @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
- * @param {number} summaryData.staleUnitsCount - Count of stale units
- * @param {number} summaryData.totalAvailable - Total available units today
- * @param {Object} logger - Logger instance
- * @returns {Promise<Object>} MongoDB updateOne result
- */
-async function updateDailySummary(db, summaryData, logger) {
-  const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
-  const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
-
-  // Calculate yesterday's date for comparison
-  const yesterday = getYesterday(date);
-
-  // Get yesterday's summary for comparison
-  const yesterdaySummary = await collection.findOne({ date: yesterday });
-  const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
-
-  const summary = {
-    date,
-    timestamp: new Date().toISOString(),
-    new_units: newUnits,
-    rented_units: rentedUnits,
-    stale_units: [], // Stale units list is not tracked per PRD
-    new_units_count: newUnits.length,
-    rented_units_count: rentedUnits.length,
-    stale_units_count: staleUnitsCount,
-    net_change: newUnits.length - rentedUnits.length,
-    total_available_today: totalAvailable,
-    total_available_yesterday: yesterdayTotal,
-    turnover_rate: yesterdayTotal > 0
-      ? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
-      : 0
-  };
-
-  try {
-    const result = await collection.updateOne(
-      { date },
-      { $set: summary },
-      { upsert: true }
-    );
-
-    logger.info('Daily summary updated', {
-      date,
-      newUnits: newUnits.length,
-      rentedUnits: rentedUnits.length,
-      totalAvailable
-    });
-
-    return result;
-
-  } catch (error) {
-    logger.error('Failed to update daily summary', {
-      errorType: error.name,
-      errorMessage: error.message
-    });
-    throw error;
-  }
-}
-
-// ============================================================
-// Additional Helpers for runScrape Orchestration
-// ============================================================
-
-/**
- * Get today's date in YYYY-MM-DD format using the configured timezone.
- * @returns {string} Today's date string
- */
-function getToday() {
-  return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
-}
-
-/**
- * Get the set of unit codes that had price records yesterday.
- * Used to calculate new and rented units by comparison.
- * @param {Db} db - MongoDB database instance
- * @param {string} today - Today's date in YYYY-MM-DD format
- * @returns {Promise<Set<string>>} Set of unit codes from yesterday
- */
-async function getYesterdayUnitCodes(db, today) {
-  const yesterday = getYesterday(today);
-  const collection = db.collection(config.COLLECTIONS.PRICES);
-
-  const yesterdayRecords = await collection
-    .find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
-    .toArray();
-
-  return new Set(yesterdayRecords.map(r => r.unit_code));
-}
-
-// ============================================================
-// Error Sanitization
-// ============================================================
-
-/**
- * Sanitize an error object to remove sensitive information before storage.
- * Removes file paths, connection strings, and credential patterns while
- * preserving the error type and a useful general description for debugging.
- *
- * @param {Error} error - Error object to sanitize
- * @returns {Object} Sanitized error with name, message, and optionally stack
- */
-function sanitizeError(error) {
-  const sanitized = {
-    name: error.name || 'Error',
-    message: sanitizeMessage(error.message || ''),
-  };
-
-  if (error.stack) {
-    sanitized.stack = sanitizeMessage(error.stack);
-  }
-
-  return sanitized;
-}
-
-/**
- * Sanitize a string message by removing sensitive patterns.
- * @param {string} message - Raw error message
- * @returns {string} Sanitized message
- */
-function sanitizeMessage(message) {
-  let result = message;
-
-  // Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
-  result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
-
-  // Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
-  result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
-
-  // Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
-  result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
-
-  // Remove Windows-style absolute paths (C:\..., D:\...)
-  result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
-
-  return result;
-}
-
-// ============================================================
-// Scraper Run History
-// ============================================================
-
-/**
- * Record scraper run to history collection.
- * Called in the finally block of runScrape() to persist run metadata.
- * This function must NOT throw errors - history recording should never break the scraper.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
- * @param {Object} logger - Logger instance
- * @returns {Promise<Object|null>} Insert result, or null on failure
- */
-async function recordScraperRun(db, runData, logger) {
-  try {
-    const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
-    const result = await collection.insertOne({
-      ...runData,
-      recordedAt: new Date()
-    });
-
-    return result;
-
-  } catch (error) {
-    // Log but don't throw - recording history should not break scraper
-    logger.error('Failed to record scraper run', { errorMessage: error.message });
-    return null;
-  }
-}
-
-// ============================================================
-// Index Management
-// ============================================================
-
-/**
- * Create indexes on the scraper_runs collection.
- * Should be called once during application startup.
- * Idempotent - safe to call multiple times.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Object} logger - Logger instance
- * @returns {Promise<void>}
- */
-async function createScraperIndexes(db, logger) {
-  const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
-
-  // Compound index for querying runs by status sorted by most recent
-  await collection.createIndex(
-    { status: 1, startedAt: -1 },
-    { name: 'status_startedAt' }
-  );
-
-  // Index for sorting all runs by start time (most recent first)
-  await collection.createIndex(
-    { startedAt: -1 },
-    { name: 'startedAt_desc' }
-  );
-
-  logger.info('Scraper indexes created successfully');
-}
-
-// ============================================================
-// Main Orchestration Function
-// ============================================================
-
-/**
- * Execute a complete scrape operation.
- * Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Object} options - Scrape options
- * @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
- * @param {string} [options.jobId] - Optional job ID (generated if not provided)
- * @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
- * @param {string} [options.htmlContent] - Use provided HTML instead of fetching
- * @returns {Promise<Object>} Scrape result with status and metrics
- */
-async function runScrape(db, options = {}) {
-  const jobId = options.jobId || crypto.randomUUID();
-  const trigger = options.trigger || 'manual';
-  const dryRun = options.dryRun || false;
-  const htmlContent = options.htmlContent || null;
-  const logger = createLogger(jobId);
-  const startTime = Date.now();
-
-  let result = {
-    jobId,
-    trigger,
-    dryRun,
-    status: 'running',
-    startedAt: new Date().toISOString(),
-    completedAt: null,
-    duration: null,
-    unitsProcessed: 0,
-    pricesInserted: 0,
-    newUnitsCount: 0,
-    rentedUnitsCount: 0,
-    staleUnitsCount: 0,
-    errors: []
-  };
-
-  try {
-    logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
-
-    // Step 1: Fetch HTML (or use provided content for testing)
-    const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
-
-    // Step 2: Parse units
-    const rawUnits = parseUnits(html, logger);
-
-    if (rawUnits.length === 0) {
-      logger.warn('No units found in HTML - possible structure change');
-      result.errors.push('No units found in HTML');
-    }
-
-    // Step 3: Convert data types
-    const units = rawUnits.map(unit => convertDataTypes(unit));
-    result.unitsProcessed = units.length;
-
-    // Step 4: Database operations
-    const today = getToday();
-
-    // Get yesterday's unit codes for comparison
-    const yesterdayUnits = await getYesterdayUnitCodes(db, today);
-
-    // Determine new and rented units
-    const currentUnitCodes = new Set(units.map(u => u.unit_code));
-    const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
-    const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
-
-    result.newUnitsCount = newUnits.length;
-    result.rentedUnitsCount = rentedUnits.length;
-
-    // Database operations (skip if dryRun)
-    if (dryRun) {
-      logger.info('Dry run mode - skipping database writes', {
-        wouldUpsert: units.length,
-        wouldInsertPrices: units.filter(u => u.price !== null).length
-      });
-      result.pricesInserted = 0;
-      result.staleUnitsCount = 0;
-    } else {
-      // Upsert units
-      await upsertUnits(db, units, logger);
-
-      // Insert prices
-      const pricesResult = await insertPrices(db, units, today, logger);
-      result.pricesInserted = pricesResult.insertedCount;
-
-      // Mark stale units
-      const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
-      result.staleUnitsCount = staleResult.modifiedCount;
-
-      // Update daily summary
-      await updateDailySummary(db, {
-        date: today,
-        newUnits: newUnits.map(u => u.unit_code),
-        rentedUnits,
-        staleUnitsCount: result.staleUnitsCount,
-        totalAvailable: units.length
-      }, logger);
-    }
-
-    result.status = 'success';
-
-  } catch (error) {
-    const cleanError = sanitizeError(error);
-    logger.error('Scrape failed', {
-      errorType: cleanError.name,
-      errorMessage: cleanError.message
-    });
-    result.status = 'failed';
-    result.errors.push(cleanError.message);
-
-  } finally {
-    result.completedAt = new Date().toISOString();
-    result.duration = Date.now() - startTime;
-
-    // Record run to history (always runs, even on failure)
-    await recordScraperRun(db, result, logger);
-
-    logger.info('Scrape completed', {
-      status: result.status,
-      duration: result.duration,
-      unitsProcessed: result.unitsProcessed,
-      pricesInserted: result.pricesInserted
-    });
-  }
-
-  return result;
-}
-
-module.exports = {
-  runScrape,
-  fetchPage,
-  parseUnits,
-  convertDataTypes,
-  upsertUnits,
-  insertPrices,
-  markStaleUnits,
-  updateDailySummary,
-  recordScraperRun,
-  createScraperIndexes,
-  // Export helpers for testing
-  getToday,
-  getYesterdayUnitCodes,
-  getYesterday,
-  parseInteger,
-  parsePositiveInteger,
-  parseIntegerOrString,
-  parsePrice,
-  parseBoolean,
-  parseFloatValue,
-  trimString,
-  isRetryableError,
-  sleep,
-  sanitizeError
-};