Update parseUnits tests to match real fixture data

The fixture file was updated with real website data (10 units) but
the tests still referenced old synthetic data (5 units with CCT-*
codes). Updated tests to use real unit codes from the fixture (W2707,
E3205, W2603) and moved edge case tests (image extraction, missing
attributes, unavailable units) to use inline HTML for isolation.

.dockerignore

@@ -0,0 +1,25 @@
+# Environment and secrets
+.env
+.env.*
+!.env.example
+
+# Dependencies
+node_modules
+
+# Git
+.git
+.gitignore
+
+# Development files
+*.log
+npm-debug.log*
+.DS_Store
+
+# Test files
+test-endpoints.js
+*.test.js
+__tests__
+
+# IDE
+.vscode
+.idea

.env.example

@@ -0,0 +1,20 @@
+# MongoDB Connection
+MONGO_URI=mongodb://username:password@host:27017
+
+# Google OAuth Credentials (from Google Cloud Console)
+GOOGLE_CLIENT_ID=your-client-id.apps.googleusercontent.com
+GOOGLE_CLIENT_SECRET=your-client-secret
+GOOGLE_CALLBACK_URL=https://apartments.maverickapplications.com/api/auth/google/callback
+
+# JWT Configuration
+# Generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
+JWT_SECRET=generate-a-secure-random-string-minimum-32-characters
+
+# Application URLs
+FRONTEND_URL=https://apartments.maverickapplications.com
+
+# Activity Logging Level: all, navigation, none
+ACTIVITY_LOG_LEVEL=all
+
+# Node Environment
+NODE_ENV=production

.github/workflows/deploy.yml

@@ -1,60 +1,372 @@
-name: Deploy Apartment API
+name: CI/CD Pipeline - Apartment API
 
 on:
   push:
     branches: [ main ]
+  pull_request:
   workflow_dispatch:
 
+env:
+  NODE_VERSION: '20'
+  IMAGE_NAME: apartment-api
+
 jobs:
-  deploy:
+  # ============================================================
+  # CI Job - Lint + Test in a single job (one checkout, one npm ci)
+  # ============================================================
+  ci:
+    name: Lint & Test
+    runs-on: ubuntu-latest
+
+    outputs:
+      lint_status: ${{ steps.lint.outcome }}
+      lint_output: ${{ steps.lint.outputs.lint_output }}
+      test_status: ${{ steps.test.outcome }}
+      test_output: ${{ steps.test.outputs.test_output }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run ESLint
+        id: lint
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm run lint 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+
+      - name: Run tests
+        id: test
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm test -- --runInBand 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "test_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+        env:
+          JWT_SECRET: test-jwt-secret-for-ci
+          NODE_ENV: test
+
+  # ============================================================
+  # Notify Job - Send results to n8n webhook
+  # ============================================================
+  notify:
+    name: Send Webhook Notification
+    runs-on: ubuntu-latest
+    needs: [ci]
+    if: always()
+
+    steps:
+      - name: Send results to n8n webhook
+        env:
+          LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
+          TEST_STATUS: ${{ needs.ci.outputs.test_status }}
+          RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
+          RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
+          GH_REPO: ${{ github.repository }}
+          GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+          GH_SHA: ${{ github.sha }}
+          GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
+          GH_ACTOR: ${{ github.actor }}
+          GH_EVENT: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
+        run: |
+          # Determine overall status
+          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
+            OVERALL_STATUS="success"
+          else
+            OVERALL_STATUS="failure"
+          fi
+
+          # Outputs are already truncated at the source (ci job)
+          LINT_OUTPUT="$RAW_LINT_OUTPUT"
+          TEST_OUTPUT="$RAW_TEST_OUTPUT"
+
+          # Build JSON payload
+          PAYLOAD=$(jq -n \
+            --arg repo "$GH_REPO" \
+            --arg branch "$GH_BRANCH" \
+            --arg commit "$GH_SHA" \
+            --arg commit_short "${GH_SHA:0:7}" \
+            --arg commit_message "$GH_COMMIT_MSG" \
+            --arg author "$GH_ACTOR" \
+            --arg event "$GH_EVENT" \
+            --arg pr_number "$GH_PR_NUMBER" \
+            --arg run_id "$GH_RUN_ID" \
+            --arg run_url "$GH_RUN_URL" \
+            --arg overall_status "$OVERALL_STATUS" \
+            --arg lint_status "$LINT_STATUS" \
+            --arg lint_output "$LINT_OUTPUT" \
+            --arg test_status "$TEST_STATUS" \
+            --arg test_output "$TEST_OUTPUT" \
+            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            '{
+              repository: $repo,
+              branch: $branch,
+              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
+              commit: {
+                sha: $commit,
+                short_sha: $commit_short,
+                message: $commit_message,
+                author: $author
+              },
+              event: $event,
+              run: {
+                id: $run_id,
+                url: $run_url
+              },
+              status: $overall_status,
+              results: {
+                lint: {
+                  status: $lint_status,
+                  output: (if $lint_status != "success" then $lint_output else null end)
+                },
+                test: {
+                  status: $test_status,
+                  output: (if $test_status != "success" then $test_output else null end)
+                }
+              },
+              timestamp: $timestamp
+            }')
+
+          # Send webhook
+          curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "$WEBHOOK_URL" \
+            --fail --silent --show-error
+
+      - name: Fail if lint or tests failed
+        if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "   Lint: ${{ needs.ci.outputs.lint_status }}"
+          echo "   Test: ${{ needs.ci.outputs.test_status }}"
+          exit 1
+
+  # ============================================================
+  # Dependency Scan Job - Runs in parallel with tests
+  # ============================================================
+  scan-deps:
+    name: Scan Dependencies
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout code
+      - name: Checkout code
-      uses: actions/checkout@v4
+        uses: actions/checkout@v4
 
-    - name: Setup Node.js
+      - name: Install Trivy
-      uses: actions/setup-node@v4
+        run: |
-      with:
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
-        node-version: '18'
-        cache: 'npm'
 
-    - name: Install dependencies
+      - name: Scan dependencies for vulnerabilities
-      run: npm ci
+        run: |
+          trivy fs \
+            --exit-code 1 \
+            --ignore-unfixed \
+            --severity CRITICAL,HIGH \
+            --scanners vuln \
+            .
 
-    - name: Build Docker image
+  # ============================================================
-      run: |
+  # Build & Push Job - Build image and push to Harbor
-        docker build -t apartment-api:${{ github.sha }} .
+  # ============================================================
-        docker tag apartment-api:${{ github.sha }} apartment-api:latest
+  build:
+    name: Build & Push Image
+    runs-on: ubuntu-latest
+    needs: [ci, scan-deps, notify]
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
 
-    - name: Deploy to server
+    outputs:
-      run: |
+      image_tag: ${{ steps.set-tag.outputs.tag }}
-        # Copy files to deployment directory
+      full_image: ${{ steps.set-tag.outputs.full_image }}
-        mkdir -p /media/stephen/Storage_Linux/infrastructure/services/apartment-api
-        cp -r . /media/stephen/Storage_Linux/infrastructure/services/apartment-api/
 
-        # Navigate to deployment directory
+    steps:
-        cd /media/stephen/Storage_Linux/infrastructure/services/apartment-api
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-        # Stop existing container if running
+      - name: Set image tag
-        docker compose down || true
+        id: set-tag
+        run: |
+          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+          echo "tag=${SHORT_SHA}" >> $GITHUB_OUTPUT
+          echo "full_image=${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${SHORT_SHA}" >> $GITHUB_OUTPUT
 
-        # Start new container
+      - name: Set up Docker Buildx
-        docker compose up -d
+        uses: docker/setup-buildx-action@v3
 
-        # Clean up old images
+      - name: Log in to Harbor
-        docker image prune -f
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.HARBOR_REGISTRY }}
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
 
-    - name: Verify deployment
+      - name: Build and push Docker image
-      run: |
+        id: build-push
-        # Wait for container to start
+        uses: docker/build-push-action@v5
-        sleep 15
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:latest
 
-        # Check if container is running
+      # ============================================================
-        docker ps | grep apartment-api
+      # Vulnerability Scanning with Trivy
+      # ============================================================
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
 
-        # Test health endpoint
+      - name: Run Trivy vulnerability scanner
-        curl -f http://localhost:3000/health || echo "API may still be starting..."
+        run: |
+          trivy image \
+            --exit-code 1 \
+            --ignore-unfixed \
+            --vuln-type os,library \
+            --severity CRITICAL,HIGH \
+            --format table \
+            ${{ steps.set-tag.outputs.full_image }}
 
-        # Test API endpoint
+      - name: Run Trivy and output SARIF
-        curl -f http://localhost:3000/api/daily-summary || echo "API may still be loading data..."
+        if: always()
+        run: |
+          trivy image \
+            --format sarif \
+            --output trivy-results.sarif \
+            ${{ steps.set-tag.outputs.full_image }} || true
+
+      # ============================================================
+      # SBOM Generation with Syft
+      # ============================================================
+      - name: Generate SBOM with Syft
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.set-tag.outputs.full_image }}
+          format: spdx-json
+          output-file: sbom.spdx.json
+
+      # ============================================================
+      # Image Signing with Cosign
+      # ============================================================
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with Cosign
+        env:
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        run: |
+          cosign sign --key env://COSIGN_PRIVATE_KEY \
+            --yes \
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
+
+      # ============================================================
+      # Attach SBOM to image in Harbor
+      # ============================================================
+      - name: Attach SBOM to image
+        env:
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        run: |
+          cosign attest --key env://COSIGN_PRIVATE_KEY \
+            --type spdxjson \
+            --predicate sbom.spdx.json \
+            --yes \
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
+
+  # ============================================================
+  # Deploy Job - Pull image and restart on production server
+  # ============================================================
+  deploy:
+    name: Deploy to Production
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+
+    steps:
+      - name: Deploy via SSH
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          port: ${{ secrets.SSH_PORT || 22 }}
+          envs: IMAGE
+          script_stop: true
+          script: |
+            set -e
+
+            # Image to deploy (using specific SHA tag, not latest)
+            IMAGE="${{ needs.build.outputs.full_image }}"
+            echo "Deploying image: $IMAGE"
+
+            # Navigate to deployment directory
+            cd ${{ secrets.DEPLOY_PATH }}
+
+            # Pull the specific image (using pre-configured Docker credentials)
+            docker pull "$IMAGE"
+
+            # Update and restart with new image
+            export IMAGE
+            docker compose up -d
+
+            # Clean up old images
+            docker image prune -f
+
+            # Wait for container to be healthy
+            echo "Waiting for container to start..."
+            sleep 10
+
+            # Verify container is running
+            if docker compose ps | grep -q "Up"; then
+              echo "Container is running successfully"
+            else
+              echo "ERROR: Container failed to start"
+              docker compose logs --tail=50
+              exit 1
+            fi
+
+      - name: Verify deployment
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          port: ${{ secrets.SSH_PORT || 22 }}
+          script: |
+            # Test health endpoint
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://apartments.maverickapplications.com/api/health || echo "000")
+
+            if [ "$HTTP_CODE" = "200" ]; then
+              echo "Health check passed (HTTP $HTTP_CODE)"
+            else
+              echo "WARNING: Health check returned HTTP $HTTP_CODE"
+              echo "Container may still be initializing..."
+              exit 1
+            fi

AUTH.md

@@ -0,0 +1,717 @@
+# Google OAuth Authentication Implementation
+
+## Overview
+
+This document details the Google OAuth authentication implementation for the Apartment Dashboard application. **All pages and API endpoints require authentication** - unauthenticated users are redirected to a login page.
+
+> **Status:** Core authentication is complete and deployed. Admin dashboard and future enhancements are documented but not yet implemented.
+
+---
+
+## Table of Contents
+
+0. [Project Context](#0-project-context)
+1. [Implementation Status](#1-implementation-status)
+2. [Authentication Scope](#2-authentication-scope)
+3. [Technical Architecture](#3-technical-architecture)
+4. [Implementation Details](#4-implementation-details)
+5. [User Activity Logging](#5-user-activity-logging)
+6. [Security Measures](#6-security-measures)
+7. [Deployment Configuration](#7-deployment-configuration)
+8. [Test Checklist](#8-test-checklist)
+9. [Future: Admin Dashboard](#9-future-admin-dashboard)
+10. [Future: Adding Apple Sign-In](#10-future-adding-apple-sign-in)
+11. [Future: Partial Public Access](#11-future-partial-public-access)
+
+---
+
+## 0. Project Context
+
+### Project Structure
+
+```
+TowersPriceWebApp/
+├── apartment-dashboard/              # Frontend (React/Vite)
+│   ├── src/
+│   │   ├── App.jsx                   # Main app (~1300 lines)
+│   │   ├── main.jsx                  # Entry point (wrapped with AuthProvider)
+│   │   ├── index.css                 # Tailwind imports
+│   │   ├── context/
+│   │   │   └── AuthContext.jsx       # Auth state management
+│   │   ├── hooks/
+│   │   │   ├── useAuth.js            # Auth hook
+│   │   │   └── useActivityTracker.js # Activity tracking hook
+│   │   ├── pages/
+│   │   │   └── Login.jsx             # Login page with Google button
+│   │   └── components/
+│   │       └── ProtectedRoute.jsx    # Route guard component
+│   ├── vite.config.js
+│   ├── package.json
+│   ├── Dockerfile
+│   └── nginx.conf
+│
+├── apartment-dashboard-api/          # Backend (Express.js)
+│   ├── server.js                     # Main server (~1700 lines)
+│   ├── config/
+│   │   └── auth.js                   # JWT, cookie, OAuth config
+│   ├── models/
+│   │   └── user.js                   # User CRUD operations
+│   ├── middleware/
+│   │   ├── passport.js               # Google OAuth strategy
+│   │   └── auth.js                   # JWT verification middleware
+│   ├── routes/
+│   │   ├── auth.js                   # OAuth routes
+│   │   └── activity.js               # Activity logging routes
+│   ├── services/
+│   │   └── activityLogger.js         # Activity logging service
+│   ├── .env.example                  # Environment template
+│   ├── .dockerignore                 # Prevents secrets in image
+│   ├── package.json
+│   ├── Dockerfile
+│   └── docker-compose.yml
+│
+└── AUTH.md                           # This file
+```
+
+### Tech Stack
+
+| Layer | Technology | Version |
+|-------|------------|---------|
+| Frontend Framework | React | 19.1.0 |
+| Frontend Build | Vite | 7.0.4 |
+| Frontend Routing | React Router DOM | 7.12.0 |
+| Frontend Styling | Tailwind CSS | 3.4.17 |
+| Frontend Charts | Recharts | 3.1.0 |
+| Backend Framework | Express.js | 4.18.2 |
+| Database | MongoDB | 6.3.0 (driver) |
+| Auth Library | Passport.js | passport-google-oauth20 |
+| Token Management | jsonwebtoken | HTTP-only cookies |
+| Reverse Proxy | Traefik | (with Let's Encrypt SSL) |
+| Static Server | Nginx | (serves built React app) |
+
+### URLs and Domains
+
+| Environment | Frontend URL | API URL |
+|-------------|--------------|---------|
+| Production | `https://apartments.maverickapplications.com` | `https://apartments.maverickapplications.com/api` |
+| Development | `http://localhost:5173` (Vite) | `http://localhost:3000` |
+
+### MongoDB Details
+
+| Setting | Value |
+|---------|-------|
+| Database name | `apartments` |
+| Data collections | `units_migration_test`, `unit_prices_migration_test`, `daily_summaries` |
+| Auth collections | `users`, `user_activity` |
+
+---
+
+## 1. Implementation Status
+
+### Completed
+
+| Feature | Status | Notes |
+|---------|--------|-------|
+| Google OAuth login | Done | Passport.js with state parameter CSRF protection |
+| JWT in HTTP-only cookies | Done | 7-day expiry with sliding window refresh |
+| User model with upsert | Done | findOneAndUpdate with $setOnInsert pattern |
+| Protected API endpoints | Done | All data endpoints require valid JWT |
+| Activity logging | Done | Configurable levels, TTL indexes for cleanup |
+| Frontend auth context | Done | React Context with useAuth hook |
+| Protected routes | Done | ProtectedRoute component with redirect |
+| Login page | Done | Google Sign-In button with error display |
+| Security hardening | Done | OAuth state, input validation, .dockerignore |
+| Docker deployment | Done | env_file configuration, production settings |
+
+### Not Yet Implemented
+
+| Feature | Priority | Notes |
+|---------|----------|-------|
+| Admin dashboard | Low | User management, activity viewer |
+| Apple Sign-In | Future | Requires Apple Developer account |
+| Partial public access | Future | Blurred preview for unauthenticated users |
+
+---
+
+## 2. Authentication Scope
+
+### What Requires Authentication
+
+**All pages and API endpoints require authentication.** Unauthenticated users see only the login page.
+
+| Page/Feature | Unauthenticated | Authenticated |
+|--------------|-----------------|---------------|
+| Login Page (`/login`) | Accessible | Redirects to `/` |
+| Overview Page (`/`) | Redirect to login | Fully accessible |
+| Units Page (`/units`) | Redirect to login | Fully accessible |
+| Analytics Page (`/analytics`) | Redirect to login | Fully accessible |
+| All API Endpoints | 401 Unauthorized | Accessible |
+
+### Protected API Endpoints
+
+```
+GET /api/health               → Public (health check only)
+GET /api/daily-summary        → Protected
+GET /api/price-history        → Protected
+GET /api/available-units      → Protected
+GET /api/recent-activity      → Protected
+GET /api/plan-stats           → Protected
+GET /api/unit/:code/history   → Protected
+GET /api/analytics            → Protected
+GET /api/best-deals           → Protected
+GET /api/price-drops          → Protected
+GET /api/stale-inventory      → Protected
+GET /api/market-insights      → Protected
+```
+
+### Auth Routes
+
+```
+GET  /auth/google             → Initiates OAuth flow (sets state cookie)
+GET  /auth/google/callback    → Handles callback (validates state, sets JWT)
+GET  /auth/me                 → Returns current user info
+GET  /auth/status             → Returns { authenticated: true/false }
+POST /auth/logout             → Clears auth cookie
+```
+
+---
+
+## 3. Technical Architecture
+
+### Authentication Flow
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                           Google OAuth Flow                                  │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│  1. User clicks "Sign in with Google"                                       │
+│           │                                                                 │
+│           ▼                                                                 │
+│  2. Frontend redirects to: /api/auth/google                                 │
+│           │                                                                 │
+│           ▼                                                                 │
+│  3. Backend generates state parameter, stores in cookie                     │
+│           │                                                                 │
+│           ▼                                                                 │
+│  4. Backend redirects to Google's OAuth consent screen                      │
+│           │                                                                 │
+│           ▼                                                                 │
+│  5. User authenticates with Google                                          │
+│           │                                                                 │
+│           ▼                                                                 │
+│  6. Google redirects to: /api/auth/google/callback?code=XXX&state=YYY       │
+│           │                                                                 │
+│           ▼                                                                 │
+│  7. Backend validates state parameter against cookie (CSRF protection)      │
+│           │                                                                 │
+│           ▼                                                                 │
+│  8. Backend exchanges code for tokens, fetches user profile                 │
+│           │                                                                 │
+│           ▼                                                                 │
+│  9. Backend creates/updates user in MongoDB (upsert)                        │
+│           │                                                                 │
+│           ▼                                                                 │
+│  10. Backend creates JWT, sets HTTP-only cookie                             │
+│           │                                                                 │
+│           ▼                                                                 │
+│  11. Backend redirects to frontend                                          │
+│           │                                                                 │
+│           ▼                                                                 │
+│  12. Frontend AuthContext checks /auth/status, renders authenticated UI     │
+│                                                                             │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+### Token Refresh Strategy (Sliding Window)
+
+Active users get their tokens refreshed automatically to avoid abrupt logouts.
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                        Sliding Window Refresh                            │
+├─────────────────────────────────────────────────────────────────────────┤
+│                                                                         │
+│  1. User makes authenticated request                                    │
+│           │                                                             │
+│           ▼                                                             │
+│  2. Auth middleware validates JWT                                       │
+│           │                                                             │
+│           ▼                                                             │
+│  3. Check: Is user still active? (isActive field)                       │
+│           │                                                             │
+│      ┌────┴────┐                                                        │
+│      │         │                                                        │
+│    No ▼       Yes ▼                                                     │
+│  Clear cookie  4. Check token age: (now - iat) > 1 day?                 │
+│  Return 401         │                                                   │
+│                ┌────┴────┐                                              │
+│                │         │                                              │
+│              No ▼       Yes ▼                                           │
+│           Continue    5. Issue new JWT with fresh 7-day expiry          │
+│           normally         │                                            │
+│                            ▼                                            │
+│                       6. Set new cookie in response                     │
+│                                                                         │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+| Scenario | Token Age | Action |
+|----------|-----------|--------|
+| User active daily | < 1 day since last refresh | No refresh |
+| User returns after 2 days | 2 days old | Refresh token |
+| User returns after 8 days | Expired | 401, redirect to login |
+
+---
+
+## 4. Implementation Details
+
+### Auth Configuration (`config/auth.js`)
+
+```javascript
+module.exports = {
+  google: {
+    clientID: process.env.GOOGLE_CLIENT_ID,
+    clientSecret: process.env.GOOGLE_CLIENT_SECRET,
+    callbackURL: process.env.GOOGLE_CALLBACK_URL,
+    scope: ['profile', 'email']
+  },
+  jwt: {
+    secret: process.env.JWT_SECRET,
+    expiresIn: '7d',
+    refreshThreshold: 24 * 60 * 60  // 1 day in seconds
+  },
+  cookie: {
+    name: 'auth_token',
+    options: {
+      httpOnly: true,
+      secure: process.env.NODE_ENV === 'production',
+      sameSite: 'lax',
+      maxAge: 7 * 24 * 60 * 60 * 1000,  // 7 days
+      domain: process.env.NODE_ENV === 'production'
+        ? '.maverickapplications.com'
+        : undefined
+    }
+  }
+};
+```
+
+### User Model (`models/user.js`)
+
+Uses MongoDB native driver with upsert pattern:
+
+```javascript
+async function findOrCreateUser(db, profile) {
+  const now = new Date();
+
+  const result = await db.collection('users').findOneAndUpdate(
+    { googleId: profile.googleId },
+    {
+      $set: {
+        email: profile.email,
+        name: profile.name,
+        picture: profile.picture || null,
+        lastLoginAt: now
+      },
+      $inc: { loginCount: 1 },
+      $setOnInsert: {
+        googleId: profile.googleId,
+        isActive: true,
+        role: 'user',
+        createdAt: now
+      }
+    },
+    { upsert: true, returnDocument: 'after' }
+  );
+
+  return result;
+}
+```
+
+**Note:** `loginCount` must NOT be in `$setOnInsert` - it conflicts with `$inc`.
+
+### Passport Strategy (`middleware/passport.js`)
+
+Safely extracts profile data from Google:
+
+```javascript
+async (accessToken, refreshToken, profile, done) => {
+  try {
+    const email = profile.emails?.[0]?.value;
+    if (!email) {
+      return done(new Error('No email found in Google profile'), null);
+    }
+
+    const userProfile = {
+      googleId: profile.id,
+      email: email,
+      name: profile.displayName,
+      picture: profile.photos?.[0]?.value || null
+    };
+
+    const user = await findOrCreateUser(db, userProfile);
+    done(null, user);
+  } catch (error) {
+    console.error('Passport strategy error:', error);
+    done(error, null);
+  }
+}
+```
+
+### OAuth State Validation (`routes/auth.js`)
+
+CSRF protection using state parameter:
+
+```javascript
+// Initiate OAuth - generate and store state
+router.get('/google', (req, res, next) => {
+  const state = crypto.randomBytes(32).toString('hex');
+  res.cookie('oauth_state', state, {
+    httpOnly: true,
+    secure: process.env.NODE_ENV === 'production',
+    sameSite: 'lax',
+    maxAge: 5 * 60 * 1000  // 5 minutes
+  });
+  passport.authenticate('google', {
+    scope: authConfig.google.scope,
+    session: false,
+    state: state
+  })(req, res, next);
+});
+
+// Callback - validate state before processing
+router.get('/google/callback', (req, res, next) => {
+  const stateFromCookie = req.cookies.oauth_state;
+  const stateFromQuery = req.query.state;
+  res.clearCookie('oauth_state');
+
+  if (!stateFromCookie || !stateFromQuery || stateFromCookie !== stateFromQuery) {
+    return res.redirect(`${process.env.FRONTEND_URL}/login?error=invalid_state`);
+  }
+  // ... proceed with authentication
+});
+```
+
+### Frontend Auth Context (`context/AuthContext.jsx`)
+
+```javascript
+const AuthContext = createContext(null);
+
+export function AuthProvider({ children }) {
+  const [user, setUser] = useState(null);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    checkAuthStatus();
+  }, []);
+
+  const checkAuthStatus = async () => {
+    try {
+      const response = await fetch(`${API_BASE}/auth/status`, {
+        credentials: 'include'
+      });
+      const data = await response.json();
+      if (data.authenticated) {
+        setUser(data.user);
+      }
+    } catch (error) {
+      console.error('Auth check failed:', error);
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  const login = () => {
+    window.location.href = `${API_BASE}/auth/google`;
+  };
+
+  const logout = async () => {
+    await fetch(`${API_BASE}/auth/logout`, {
+      method: 'POST',
+      credentials: 'include'
+    });
+    setUser(null);
+  };
+
+  return (
+    <AuthContext.Provider value={{ user, loading, login, logout }}>
+      {children}
+    </AuthContext.Provider>
+  );
+}
+```
+
+### Database Indexes
+
+Created automatically on server startup:
+
+**Users Collection:**
+- `{ googleId: 1 }` - unique
+- `{ email: 1 }` - unique
+- `{ isActive: 1, lastLoginAt: -1 }` - compound for queries
+
+**User Activity Collection:**
+- `{ userId: 1, timestamp: -1 }` - user activity queries
+- `{ timestamp: 1 }` - TTL index (90-day auto-cleanup)
+- `{ action: 1, timestamp: -1 }` - action type queries
+- `{ sessionId: 1 }` - session grouping
+
+---
+
+## 5. User Activity Logging
+
+### Configuration
+
+Controlled by `ACTIVITY_LOG_LEVEL` environment variable:
+
+| Level | What's Logged |
+|-------|---------------|
+| `all` | All user interactions (default) |
+| `navigation` | Only page views and route changes |
+| `none` | Logging disabled |
+
+### Activity Types
+
+| Action | When Logged | Metadata |
+|--------|-------------|----------|
+| `LOGIN` | User completes OAuth | `{ method: 'google' }` |
+| `LOGOUT` | User logs out | `{}` |
+| `PAGE_VIEW` | Page load | `{ path }` |
+| `NAVIGATION` | Route change | `{ from, to }` |
+
+### Activity Schema
+
+```javascript
+{
+  _id: ObjectId,
+  userId: ObjectId,
+  sessionId: String,      // UUID for grouping
+  action: String,
+  metadata: Object,
+  page: String,
+  timestamp: Date,
+  userAgent: String,
+  ip: String
+}
+```
+
+### Auto-Cleanup
+
+Activity records are automatically deleted after 90 days via MongoDB TTL index.
+
+---
+
+## 6. Security Measures
+
+### Implemented
+
+| Security Feature | Implementation |
+|------------------|----------------|
+| OAuth state parameter | Random 32-byte hex, stored in cookie, validated on callback |
+| HTTP-only cookies | JWT not accessible via JavaScript |
+| Secure cookies (production) | Only sent over HTTPS |
+| SameSite=Lax | CSRF protection for cookies |
+| Input validation | Unit codes validated with regex pattern |
+| isActive check | Disabled users rejected on every request |
+| .dockerignore | Prevents .env and secrets from being copied into images |
+| Environment validation | Server exits if MONGO_URI missing in production |
+
+### Input Validation
+
+```javascript
+const isValidUnitCode = (unitCode) => {
+  return typeof unitCode === 'string' &&
+         unitCode.length > 0 &&
+         unitCode.length <= 20 &&
+         /^[A-Za-z0-9_-]+$/.test(unitCode);
+};
+```
+
+### CORS Configuration
+
+```javascript
+const corsOptions = {
+  origin: process.env.FRONTEND_URL,  // Exact origin, not '*'
+  credentials: true,                  // Required for cookies
+  methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
+  allowedHeaders: ['Content-Type', 'Authorization'],
+  exposedHeaders: ['set-cookie']
+};
+```
+
+---
+
+## 7. Deployment Configuration
+
+### Environment Variables
+
+Backend `.env` (see `.env.example` for template):
+
+```env
+# MongoDB
+MONGO_URI=mongodb+srv://...
+
+# Server
+PORT=8080
+NODE_ENV=production
+
+# Google OAuth
+GOOGLE_CLIENT_ID=xxx.apps.googleusercontent.com
+GOOGLE_CLIENT_SECRET=xxx
+GOOGLE_CALLBACK_URL=https://apartments.maverickapplications.com/api/auth/google/callback
+
+# JWT (generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))")
+JWT_SECRET=your-secure-random-string
+
+# App
+FRONTEND_URL=https://apartments.maverickapplications.com
+
+# Activity Logging
+ACTIVITY_LOG_LEVEL=all
+```
+
+### Docker Compose
+
+```yaml
+services:
+  apartment-api:
+    build: .
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=production
+```
+
+### Google Cloud Console Setup
+
+Required redirect URIs:
+```
+https://apartments.maverickapplications.com/api/auth/google/callback
+```
+
+OAuth consent screen:
+- User type: External
+- Scopes: `email`, `profile` (non-sensitive)
+
+---
+
+## 8. Test Checklist
+
+### Completed Tests
+
+- [x] Google OAuth login flow works end-to-end
+- [x] JWT cookie is set with correct flags (httpOnly, secure, sameSite)
+- [x] Protected endpoints return 401 without valid token
+- [x] User data persists across sessions (cookie survives browser close)
+- [x] Activity logging captures LOGIN/LOGOUT events
+- [x] Data loads correctly regardless of server timezone
+- [x] OAuth state parameter prevents CSRF
+- [x] User upsert creates new users and updates existing
+- [x] Sliding window token refresh extends sessions
+- [x] Logout clears cookie and redirects to login
+
+### Production Verification
+
+- [x] OAuth redirect URIs match production domain
+- [x] HTTPS enforced
+- [x] Environment variables properly set
+- [x] MongoDB indexes created on startup
+- [x] No sensitive data in console logs
+- [x] .dockerignore prevents secrets in image
+
+---
+
+## 9. Future: Admin Dashboard
+
+**Priority: Low**
+
+### Planned Features
+
+1. **User Management**
+   - View all registered users
+   - See last login, login count
+   - Enable/disable users
+
+2. **Activity Viewer**
+   - Recent activity stream
+   - Filter by user, action, date
+
+3. **Usage Statistics**
+   - Active users (daily/weekly/monthly)
+   - Most common actions
+   - Peak usage times
+
+### Access Control
+
+- Add `role: 'admin'` to user document
+- Create `requireAdmin` middleware
+- Manually set initial admin via database
+
+---
+
+## 10. Future: Adding Apple Sign-In
+
+**Priority: Future**
+
+### Requirements
+
+1. Apple Developer account ($99/year)
+2. Service ID for web authentication
+3. Private key for token verification
+
+### Changes Needed
+
+1. Install `passport-apple`
+2. Add Apple strategy to passport
+3. Add routes: `/auth/apple`, `/auth/apple/callback`
+4. Update user model for multiple providers
+5. Add "Sign in with Apple" button
+
+---
+
+## 11. Future: Partial Public Access
+
+**Priority: Future**
+
+Allow unauthenticated users to see a blurred preview of the overview page.
+
+### Behavior
+
+- Summary cards visible
+- Charts and detailed data blurred with CSS
+- Overlay with "Sign in to view" prompt
+
+### Implementation
+
+1. Make `/api/daily-summary` public
+2. Create `AuthBlurOverlay` component
+3. Wrap protected sections on overview page
+
+---
+
+## Decisions Log
+
+| Decision | Choice | Rationale |
+|----------|--------|-----------|
+| Token storage | HTTP-only cookie | More secure than localStorage |
+| Token expiry | 7 days with sliding refresh | Balance security and UX |
+| Disabled user handling | Silent logout | No error messaging needed |
+| State parameter | Random 32-byte hex | Industry standard CSRF protection |
+| Activity cleanup | 90-day TTL | Automatic via MongoDB index |
+| Date handling | Use latest database date | Handles server timezone mismatch |
+| User upsert | findOneAndUpdate | Atomic create/update operation |
+
+---
+
+## Troubleshooting
+
+### Common Issues
+
+| Issue | Cause | Solution |
+|-------|-------|----------|
+| 401 on all endpoints | Missing/invalid JWT | Check cookie is set, not expired |
+| OAuth callback 500 | Profile field access error | Use optional chaining on profile fields |
+| Empty data arrays | Timezone mismatch | Server uses latest date from database |
+| MongoDB conflict error | loginCount in $setOnInsert | Remove from $setOnInsert, use only $inc |
+| Cookie not set | CORS misconfiguration | Ensure credentials: 'include' on all fetches |
+| State validation fails | Cookie expired or cleared | State cookie has 5-minute lifetime |

Dockerfile

@@ -1,8 +1,12 @@
-FROM node:18-alpine
+FROM node:20-alpine
 
 # Set working directory
 WORKDIR /app
 
+# Update Alpine packages and npm to fix security vulnerabilities
+RUN apk upgrade --no-cache && \
+    npm install -g npm@latest
+
 # Install Curl
 RUN apk add --no-cache curl
 
@@ -10,7 +14,7 @@ RUN apk add --no-cache curl
 COPY package*.json ./
 
 # Install dependencies
-RUN npm ci --only=production
+RUN npm ci --omit=dev
 
 # Copy source code
 COPY . .

__tests__/helpers/testHelpers.js

@@ -0,0 +1,156 @@
+const { MongoClient, ObjectId } = require('mongodb');
+const jwt = require('jsonwebtoken');
+
+// Counter for generating unique identifiers within the same millisecond
+let uniqueCounter = 0;
+
+/**
+ * Create a test Express app instance with database connection
+ * This creates a minimal Express app for testing admin routes
+ */
+async function createTestApp(db) {
+  const express = require('express');
+  const cookieParser = require('cookie-parser');
+
+  const app = express();
+  app.use(express.json());
+  app.use(cookieParser());
+
+  // Store db in app.locals for middleware access
+  app.locals.db = db;
+
+  // Mount the auth middleware module
+  const { requireAuth, requireAdmin } = require('../../middleware/auth');
+
+  // Health check endpoint (should remain public)
+  app.get('/api/health', (req, res) => {
+    res.json({ status: 'ok' });
+  });
+
+  // Try to mount admin routes if they exist
+  try {
+    const adminRoutes = require('../../routes/admin');
+    app.use('/api/admin', requireAuth, requireAdmin, adminRoutes);
+  } catch (error) {
+    // Admin routes don't exist yet - this is expected for failing tests
+    // Create placeholder routes that will 404
+    app.use('/api/admin', requireAuth, requireAdmin, (req, res) => {
+      res.status(404).json({ error: 'Admin routes not implemented' });
+    });
+  }
+
+  return app;
+}
+
+/**
+ * Generate a valid JWT token for a test user
+ * @param {string|ObjectId} userId - The user's MongoDB _id
+ * @returns {string} JWT token
+ */
+function generateTestToken(userId) {
+  const secret = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
+  return jwt.sign(
+    { userId: userId.toString() },
+    secret,
+    { expiresIn: '7d' }
+  );
+}
+
+/**
+ * Create a test user document
+ * @param {Object} overrides - Fields to override in the default user
+ * @returns {Object} User document
+ */
+function createTestUser(overrides = {}) {
+  const now = new Date();
+  const uniqueId = `${Date.now()}-${++uniqueCounter}-${Math.random().toString(36).substr(2, 9)}`;
+  return {
+    _id: new ObjectId(),
+    googleId: `google-${uniqueId}`,
+    email: `test-${uniqueId}@example.com`,
+    name: 'Test User',
+    picture: 'https://example.com/photo.jpg',
+    role: 'user',
+    isActive: true,
+    loginCount: 1,
+    createdAt: now,
+    lastLoginAt: now,
+    disabledAt: null,
+    disabledBy: null,
+    ...overrides
+  };
+}
+
+/**
+ * Create an admin user document
+ * @param {Object} overrides - Fields to override in the default admin
+ * @returns {Object} Admin user document
+ */
+function createTestAdmin(overrides = {}) {
+  const uniqueId = `${Date.now()}-${++uniqueCounter}-${Math.random().toString(36).substr(2, 9)}`;
+  return createTestUser({
+    role: 'admin',
+    email: `admin-${uniqueId}@example.com`,
+    name: 'Test Admin',
+    ...overrides
+  });
+}
+
+/**
+ * Insert a user into the database
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} user - User document to insert
+ * @returns {Promise<Object>} Inserted user with _id
+ */
+async function insertTestUser(db, user) {
+  const result = await db.collection('users').insertOne(user);
+  return { ...user, _id: result.insertedId };
+}
+
+/**
+ * Clean up test users from the database
+ * @param {Db} db - MongoDB database instance
+ */
+async function cleanupTestUsers(db) {
+  await db.collection('users').deleteMany({});
+}
+
+/**
+ * Clean up all test data from the database
+ * @param {Db} db - MongoDB database instance
+ */
+async function cleanupTestData(db) {
+  await Promise.all([
+    db.collection('users').deleteMany({}),
+    db.collection('user_activity').deleteMany({})
+  ]);
+}
+
+/**
+ * Create a test activity document
+ * @param {Object} overrides - Fields to override in the default activity
+ * @returns {Object} Activity document
+ */
+function createTestActivity(overrides = {}) {
+  const now = new Date();
+  return {
+    _id: new ObjectId(),
+    userId: new ObjectId(),
+    action: 'PAGE_VIEW',
+    metadata: { path: '/test' },
+    timestamp: now,
+    sessionId: `session-${Date.now()}`,
+    ...overrides
+  };
+}
+
+module.exports = {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  insertTestUser,
+  cleanupTestUsers,
+  cleanupTestData,
+  createTestActivity
+};

__tests__/phase1-foundation.test.js

@@ -0,0 +1,905 @@
+/**
+ * Phase 1: Foundation Tests for Admin Dashboard
+ *
+ * These tests verify the implementation of Phase 1 requirements from ADMIN.md:
+ * - 1.1 Database Schema (DB-1.x): User role field, disabledAt/disabledBy fields, indexes
+ * - 1.2 Middleware (MW-1.x): requireAdmin middleware functionality
+ * - 1.3 User Management API (API-1.x): CRUD operations for admin user management
+ *
+ * These tests are designed to FAIL initially as the implementation does not exist yet.
+ * Run with: npm test
+ */
+
+const request = require('supertest');
+const { MongoClient, ObjectId } = require('mongodb');
+const {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  insertTestUser,
+  cleanupTestUsers,
+  cleanupTestData
+} = require('./helpers/testHelpers');
+
+describe('Phase 1: Foundation', () => {
+  let connection;
+  let db;
+  let app;
+
+  beforeAll(async () => {
+    // Connect to the in-memory MongoDB instance
+    const uri = process.env.MONGO_URI;
+    connection = await MongoClient.connect(uri);
+    db = connection.db('apartments_test');
+  });
+
+  afterAll(async () => {
+    if (connection) {
+      await connection.close();
+    }
+  });
+
+  beforeEach(async () => {
+    // Clean up test data before each test
+    await cleanupTestData(db);
+    // Create fresh app instance for each test
+    app = await createTestApp(db);
+  });
+
+  // =============================================================================
+  // 1.1 DATABASE SCHEMA TESTS
+  // These tests verify the user schema has been properly updated for admin features
+  // =============================================================================
+  describe('1.1 Database Schema', () => {
+    /**
+     * DB-1.1: Users must have a `role` field
+     * Valid values: 'user' | 'admin'
+     * Default: 'user'
+     */
+    describe('DB-1.1: User role field', () => {
+      it('should have role field with default value "user" for new users', async () => {
+        // When a new user is created without specifying role
+        const user = createTestUser();
+        delete user.role; // Remove role to test default
+
+        const { findOrCreateUser } = require('../models/user');
+
+        // Create a user profile to simulate OAuth flow
+        const profile = {
+          googleId: user.googleId,
+          email: user.email,
+          name: user.name,
+          picture: user.picture
+        };
+
+        const createdUser = await findOrCreateUser(db, profile);
+
+        // Then the role should default to 'user'
+        expect(createdUser).toBeDefined();
+        expect(createdUser.role).toBe('user');
+      });
+
+      it('should only allow "user" or "admin" as valid role values', async () => {
+        // Attempt to insert a user with an invalid role
+        const userWithInvalidRole = createTestUser({ role: 'superadmin' });
+
+        // This test verifies schema validation exists
+        // The exact implementation depends on whether validation is done at
+        // the application level or database level
+        await expect(async () => {
+          await db.collection('users').insertOne(userWithInvalidRole);
+          // Query the user back to verify role validation
+          const inserted = await db.collection('users').findOne({ _id: userWithInvalidRole._id });
+          // If no validation exists, this should be caught by application logic
+          if (inserted.role !== 'user' && inserted.role !== 'admin') {
+            throw new Error('Invalid role should not be allowed');
+          }
+        }).rejects.toThrow();
+      });
+    });
+
+    /**
+     * DB-1.2: Users must have `disabledAt` field
+     * Type: Date | null
+     * Set when user is disabled, null when active
+     */
+    describe('DB-1.2: User disabledAt field', () => {
+      it('should have disabledAt field set to null for active users', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        const foundUser = await db.collection('users').findOne({ _id: user._id });
+
+        expect(foundUser.disabledAt).toBeNull();
+      });
+
+      it('should have disabledAt field set to Date when user is disabled', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        // Import the user model function that handles disabling
+        const { setUserActive } = require('../models/user');
+
+        // Disable the user (this should set disabledAt)
+        const disabledUser = await setUserActive(db, user._id, false);
+
+        expect(disabledUser.isActive).toBe(false);
+        expect(disabledUser.disabledAt).toBeInstanceOf(Date);
+      });
+    });
+
+    /**
+     * DB-1.3: Users must have `disabledBy` field
+     * Type: ObjectId | null
+     * References the admin who disabled the user
+     */
+    describe('DB-1.3: User disabledBy field', () => {
+      it('should have disabledBy field set to null for active users', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        const foundUser = await db.collection('users').findOne({ _id: user._id });
+
+        expect(foundUser.disabledBy).toBeNull();
+      });
+
+      it('should have disabledBy field set to admin ObjectId when disabled', async () => {
+        const admin = createTestAdmin();
+        const user = createTestUser();
+        await insertTestUser(db, admin);
+        await insertTestUser(db, user);
+
+        // Import the updated user model function that accepts disabledBy
+        const { setUserActive } = require('../models/user');
+
+        // Disable the user with admin reference
+        const disabledUser = await setUserActive(db, user._id, false, admin._id);
+
+        expect(disabledUser.isActive).toBe(false);
+        expect(disabledUser.disabledBy).toEqual(admin._id);
+      });
+    });
+
+    /**
+     * DB-1.4: Required indexes for admin queries
+     * Indexes needed:
+     * - { role: 1 }
+     * - { isActive: 1, role: 1 }
+     * - { createdAt: -1 }
+     * - { lastLoginAt: -1 }
+     */
+    describe('DB-1.4: Required indexes', () => {
+      beforeEach(async () => {
+        // Create indexes (this should be done by createIndexes function)
+        const { createIndexes } = require('../models/user');
+        await createIndexes(db);
+      });
+
+      it('should have index on role field', async () => {
+        const indexes = await db.collection('users').indexes();
+        const roleIndex = indexes.find(idx =>
+          idx.key && idx.key.role === 1 && Object.keys(idx.key).length === 1
+        );
+
+        expect(roleIndex).toBeDefined();
+      });
+
+      it('should have compound index on isActive and role', async () => {
+        const indexes = await db.collection('users').indexes();
+        const compoundIndex = indexes.find(idx =>
+          idx.key && idx.key.isActive === 1 && idx.key.role === 1
+        );
+
+        expect(compoundIndex).toBeDefined();
+      });
+
+      it('should have index on createdAt descending', async () => {
+        const indexes = await db.collection('users').indexes();
+        const createdAtIndex = indexes.find(idx =>
+          idx.key && idx.key.createdAt === -1
+        );
+
+        expect(createdAtIndex).toBeDefined();
+      });
+
+      it('should have index on lastLoginAt descending', async () => {
+        const indexes = await db.collection('users').indexes();
+        const lastLoginIndex = indexes.find(idx =>
+          idx.key && idx.key.lastLoginAt === -1
+        );
+
+        expect(lastLoginIndex).toBeDefined();
+      });
+    });
+  });
+
+  // =============================================================================
+  // 1.2 MIDDLEWARE TESTS
+  // These tests verify the requireAdmin middleware behaves correctly
+  // =============================================================================
+  describe('1.2 Middleware', () => {
+    /**
+     * MW-1.1: requireAdmin middleware must exist
+     */
+    describe('MW-1.1: requireAdmin middleware exists', () => {
+      it('should export requireAdmin middleware from auth module', () => {
+        const { requireAdmin } = require('../middleware/auth');
+
+        expect(requireAdmin).toBeDefined();
+        expect(typeof requireAdmin).toBe('function');
+      });
+    });
+
+    /**
+     * MW-1.2: requireAdmin returns 403 if user.role !== 'admin'
+     */
+    describe('MW-1.2: requireAdmin authorization', () => {
+      it('should return 403 with "Admin access required" for non-admin users', async () => {
+        // Create a regular user (not admin)
+        const user = createTestUser({ role: 'user' });
+        await insertTestUser(db, user);
+        const token = generateTestToken(user._id);
+
+        // Attempt to access an admin endpoint
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(403);
+        expect(response.body.error).toBe('Admin access required');
+      });
+
+      it('should allow access for users with admin role', async () => {
+        // Create an admin user
+        const admin = createTestAdmin();
+        await insertTestUser(db, admin);
+        const token = generateTestToken(admin._id);
+
+        // Access admin endpoint - should not get 403
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        // Should either succeed (200) or 404 if routes not implemented
+        // but NOT 403 (forbidden)
+        expect(response.status).not.toBe(403);
+      });
+    });
+
+    /**
+     * MW-1.3: requireAdmin must work after requireAuth
+     */
+    describe('MW-1.3: Middleware chaining', () => {
+      it('should return 401 if no authentication token provided', async () => {
+        const response = await request(app)
+          .get('/api/admin/users');
+
+        expect(response.status).toBe(401);
+        expect(response.body.error).toBe('Authentication required');
+      });
+
+      it('should return 401 for invalid token before checking admin role', async () => {
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', ['auth_token=invalid-token']);
+
+        expect(response.status).toBe(401);
+      });
+
+      it('should return 401 for disabled users even if they have admin role', async () => {
+        // Create a disabled admin
+        const disabledAdmin = createTestAdmin({ isActive: false });
+        await insertTestUser(db, disabledAdmin);
+        const token = generateTestToken(disabledAdmin._id);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(401);
+      });
+    });
+
+    /**
+     * MW-1.4: /api/health must remain public
+     */
+    describe('MW-1.4: Health endpoint remains public', () => {
+      it('should return 200 on /api/health without authentication', async () => {
+        const response = await request(app)
+          .get('/api/health');
+
+        expect(response.status).toBe(200);
+        expect(response.body.status).toBe('ok');
+      });
+
+      it('should not require admin role for /api/health', async () => {
+        // Regular user accessing health endpoint
+        const user = createTestUser({ role: 'user' });
+        await insertTestUser(db, user);
+        const token = generateTestToken(user._id);
+
+        const response = await request(app)
+          .get('/api/health')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(200);
+      });
+    });
+  });
+
+  // =============================================================================
+  // 1.3 USER MANAGEMENT API TESTS
+  // These tests verify the admin user management endpoints
+  // =============================================================================
+  describe('1.3 User Management API', () => {
+    let adminUser;
+    let adminToken;
+
+    beforeEach(async () => {
+      // Create an admin user for testing admin endpoints
+      adminUser = createTestAdmin();
+      await insertTestUser(db, adminUser);
+      adminToken = generateTestToken(adminUser._id);
+    });
+
+    /**
+     * API-1.1: GET /api/admin/users - Paginated user list
+     */
+    describe('API-1.1: GET /api/admin/users', () => {
+      it('should return paginated list of users', async () => {
+        // Create some test users
+        for (let i = 0; i < 25; i++) {
+          await insertTestUser(db, createTestUser({
+            email: `user${i}@example.com`,
+            name: `User ${i}`
+          }));
+        }
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users).toBeDefined();
+        expect(Array.isArray(response.body.data.users)).toBe(true);
+        expect(response.body.data.pagination).toBeDefined();
+        expect(response.body.data.pagination.page).toBe(1);
+        expect(response.body.data.pagination.limit).toBe(20);
+        expect(response.body.data.pagination.total).toBeGreaterThan(0);
+        expect(response.body.data.pagination.pages).toBeDefined();
+      });
+
+      it('should support page parameter', async () => {
+        // Create 30 test users
+        for (let i = 0; i < 30; i++) {
+          await insertTestUser(db, createTestUser({
+            email: `user${i}@example.com`
+          }));
+        }
+
+        const response = await request(app)
+          .get('/api/admin/users?page=2')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.page).toBe(2);
+      });
+
+      it('should support limit parameter with max 100', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?limit=50')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.limit).toBe(50);
+      });
+
+      it('should cap limit at 100', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?limit=200')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.limit).toBeLessThanOrEqual(100);
+      });
+
+      it('should support search parameter for name', async () => {
+        await insertTestUser(db, createTestUser({ name: 'John Smith' }));
+        await insertTestUser(db, createTestUser({ name: 'Jane Doe' }));
+
+        const response = await request(app)
+          .get('/api/admin/users?search=John')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.some(u => u.name.includes('John'))).toBe(true);
+        expect(response.body.data.users.every(u =>
+          u.name.toLowerCase().includes('john') ||
+          u.email.toLowerCase().includes('john')
+        )).toBe(true);
+      });
+
+      it('should support search parameter for email', async () => {
+        await insertTestUser(db, createTestUser({ email: 'unique-test@example.com' }));
+
+        const response = await request(app)
+          .get('/api/admin/users?search=unique-test')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.some(u => u.email.includes('unique-test'))).toBe(true);
+      });
+
+      it('should support status filter "active"', async () => {
+        await insertTestUser(db, createTestUser({ isActive: true }));
+        await insertTestUser(db, createTestUser({ isActive: false }));
+
+        const response = await request(app)
+          .get('/api/admin/users?status=active')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.every(u => u.isActive === true)).toBe(true);
+      });
+
+      it('should support status filter "disabled"', async () => {
+        await insertTestUser(db, createTestUser({ isActive: false }));
+
+        const response = await request(app)
+          .get('/api/admin/users?status=disabled')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.every(u => u.isActive === false)).toBe(true);
+      });
+
+      it('should support sort parameter', async () => {
+        const oldUser = createTestUser({
+          createdAt: new Date('2023-01-01'),
+          email: 'old@example.com'
+        });
+        const newUser = createTestUser({
+          createdAt: new Date('2024-01-01'),
+          email: 'new@example.com'
+        });
+        await insertTestUser(db, oldUser);
+        await insertTestUser(db, newUser);
+
+        const response = await request(app)
+          .get('/api/admin/users?sort=createdAt&order=desc')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        // Newest first when sorted descending
+        const createdDates = response.body.data.users.map(u => new Date(u.createdAt));
+        for (let i = 0; i < createdDates.length - 1; i++) {
+          expect(createdDates[i] >= createdDates[i + 1]).toBe(true);
+        }
+      });
+
+      it('should support order parameter "asc" and "desc"', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?sort=loginCount&order=asc')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        // Verify ascending order
+        const counts = response.body.data.users.map(u => u.loginCount);
+        for (let i = 0; i < counts.length - 1; i++) {
+          expect(counts[i] <= counts[i + 1]).toBe(true);
+        }
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, regularUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${userToken}`]);
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.2: GET /api/admin/users/:id - User details with recent activity
+     */
+    describe('API-1.2: GET /api/admin/users/:id', () => {
+      it('should return user details for valid user ID', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user._id.toString()).toBe(testUser._id.toString());
+        expect(response.body.data.user.email).toBe(testUser.email);
+        expect(response.body.data.user.name).toBe(testUser.name);
+        expect(response.body.data.user.role).toBe(testUser.role);
+      });
+
+      it('should include recent activity for the user', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        // Create some activity for this user
+        await db.collection('user_activity').insertMany([
+          {
+            userId: testUser._id,
+            action: 'PAGE_VIEW',
+            metadata: { path: '/dashboard' },
+            timestamp: new Date(),
+            sessionId: 'session-1'
+          },
+          {
+            userId: testUser._id,
+            action: 'LOGIN',
+            timestamp: new Date(),
+            sessionId: 'session-1'
+          }
+        ]);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user.recentActivity).toBeDefined();
+        expect(Array.isArray(response.body.data.user.recentActivity)).toBe(true);
+      });
+
+      it('should return 404 for non-existent user ID', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .get(`/api/admin/users/${nonExistentId}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(404);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should return 400 for invalid user ID format', async () => {
+        const response = await request(app)
+          .get('/api/admin/users/invalid-id')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, regularUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${regularUser._id}`)
+          .set('Cookie', [`auth_token=${userToken}`]);
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.3: PATCH /api/admin/users/:id - Enable/disable user
+     */
+    describe('API-1.3: PATCH /api/admin/users/:id (enable/disable)', () => {
+      it('should disable a user successfully', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.isActive).toBe(false);
+        expect(response.body.data.message).toContain('disabled');
+
+        // Verify in database
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.isActive).toBe(false);
+      });
+
+      it('should enable a disabled user successfully', async () => {
+        const testUser = createTestUser({ isActive: false });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: true });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.isActive).toBe(true);
+        expect(response.body.data.message).toContain('enabled');
+      });
+
+      it('should set disabledAt timestamp when disabling', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const beforeDisable = new Date();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledAt).toBeInstanceOf(Date);
+        expect(dbUser.disabledAt.getTime()).toBeGreaterThanOrEqual(beforeDisable.getTime());
+      });
+
+      it('should set disabledBy to admin ID when disabling', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledBy).toEqual(adminUser._id);
+      });
+
+      it('should clear disabledAt and disabledBy when enabling', async () => {
+        const testUser = createTestUser({
+          isActive: false,
+          disabledAt: new Date(),
+          disabledBy: new ObjectId()
+        });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: true });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledAt).toBeNull();
+        expect(dbUser.disabledBy).toBeNull();
+      });
+
+      it('should return 400 when trying to disable yourself', async () => {
+        const response = await request(app)
+          .patch(`/api/admin/users/${adminUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toContain('yourself');
+      });
+
+      it('should return 404 for non-existent user', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${nonExistentId}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(404);
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        const targetUser = createTestUser();
+        await insertTestUser(db, regularUser);
+        await insertTestUser(db, targetUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${targetUser._id}`)
+          .set('Cookie', [`auth_token=${userToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.4: PATCH /api/admin/users/:id/role - Promote/demote user role
+     */
+    describe('API-1.4: PATCH /api/admin/users/:id/role (promote/demote)', () => {
+      it('should promote a user to admin', async () => {
+        const testUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.role).toBe('admin');
+        expect(response.body.data.message).toContain('promoted');
+
+        // Verify in database
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.role).toBe('admin');
+      });
+
+      it('should demote an admin to user', async () => {
+        const anotherAdmin = createTestAdmin({ email: 'another@admin.com' });
+        await insertTestUser(db, anotherAdmin);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${anotherAdmin._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'user' });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.role).toBe('user');
+        expect(response.body.data.message).toContain('demoted');
+      });
+
+      it('should return 400 when trying to demote yourself from admin', async () => {
+        const response = await request(app)
+          .patch(`/api/admin/users/${adminUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'user' });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toContain('yourself');
+      });
+
+      it('should return 400 for invalid role value', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'superadmin' });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should return 400 when role is not provided', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+
+        expect(response.status).toBe(400);
+      });
+
+      it('should return 404 for non-existent user', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${nonExistentId}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(404);
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        const targetUser = createTestUser();
+        await insertTestUser(db, regularUser);
+        await insertTestUser(db, targetUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${targetUser._id}/role`)
+          .set('Cookie', [`auth_token=${userToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(403);
+      });
+
+      it('should only allow "user" or "admin" roles', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const invalidRoles = ['superuser', 'moderator', 'guest', '', null, 123];
+
+        for (const invalidRole of invalidRoles) {
+          const response = await request(app)
+            .patch(`/api/admin/users/${testUser._id}/role`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ role: invalidRole });
+
+          expect(response.status).toBe(400);
+        }
+      });
+    });
+
+    /**
+     * API-1.5: General API security tests
+     */
+    describe('API-1.5: API Security', () => {
+      it('should not expose sensitive user data in list response', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+
+        // Check that sensitive fields are not exposed
+        const userInResponse = response.body.data.users.find(
+          u => u._id.toString() === testUser._id.toString()
+        );
+
+        if (userInResponse) {
+          // googleId should potentially be hidden or sanitized in responses
+          // This depends on your security requirements
+          expect(userInResponse.password).toBeUndefined();
+        }
+      });
+
+      it('should validate ObjectId format in URL parameters', async () => {
+        const invalidIds = ['123', 'abc', '!@#$%', '   ', ''];
+
+        for (const invalidId of invalidIds) {
+          const response = await request(app)
+            .get(`/api/admin/users/${invalidId}`)
+            .set('Cookie', [`auth_token=${adminToken}`]);
+
+          expect(response.status).toBe(400);
+        }
+      });
+
+      it('should handle concurrent requests safely', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        // Send multiple concurrent disable/enable requests
+        const requests = [
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: false }),
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: true }),
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: false })
+        ];
+
+        const responses = await Promise.all(requests);
+
+        // All requests should complete without errors (200) or with consistent state
+        responses.forEach(response => {
+          expect([200, 400, 404, 409]).toContain(response.status);
+        });
+
+        // Final state should be consistent
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(typeof dbUser.isActive).toBe('boolean');
+      });
+    });
+  });
+});

__tests__/phase4-settings-security.test.js

@@ -0,0 +1,980 @@
+/**
+ * Phase 4: Settings & Security Tests
+ *
+ * Tests for Admin Dashboard Phase 4 implementation as specified in ADMIN.md
+ * Reference: Implementation Phases -> Phase 4: Settings & Polish
+ *
+ * Endpoints tested:
+ * - GET /api/admin/settings (API-4.1)
+ * - PATCH /api/admin/settings (API-4.2, API-4.3)
+ *
+ * Security features tested:
+ * - Admin action audit logging (SEC-4.1)
+ * - Last admin protection (SEC-4.2)
+ * - Rate limiting on admin endpoints (SEC-4.3)
+ *
+ * These tests are designed to FAIL initially as the endpoints do not exist yet.
+ * They serve as the specification for implementing the Settings & Polish features.
+ */
+
+const request = require('supertest');
+const { MongoClient, ObjectId } = require('mongodb');
+const {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  cleanupTestData
+} = require('./helpers/testHelpers');
+
+// Test configuration - uses environment set by global setup
+const TEST_JWT_SECRET = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
+
+let app;
+let db;
+let client;
+
+// Test user fixtures - created fresh for each test
+let testUsers = {};
+
+/**
+ * Helper to make authenticated requests
+ * @param {string} method - HTTP method (get, post, patch, delete)
+ * @param {string} url - Request URL
+ * @param {Object} user - User to authenticate as
+ * @returns {Object} Supertest request
+ */
+const authenticatedRequest = (method, url, user) => {
+  const token = generateTestToken(user._id);
+  return request(app)[method](url)
+    .set('Cookie', `auth_token=${token}`);
+};
+
+/**
+ * Helper object for cleaner authenticated request syntax
+ * @param {Object} user - User to authenticate as
+ * @returns {Object} Object with HTTP method functions
+ */
+const authAs = (user) => ({
+  get: (url) => authenticatedRequest('get', url, user),
+  post: (url) => authenticatedRequest('post', url, user),
+  patch: (url) => authenticatedRequest('patch', url, user),
+  delete: (url) => authenticatedRequest('delete', url, user)
+});
+
+describe('Phase 4: Settings & Security', () => {
+  beforeAll(async () => {
+    // Connect to test database (MongoDB Memory Server from global setup)
+    const mongoUri = process.env.MONGO_URI || 'mongodb://localhost:27017';
+    client = new MongoClient(mongoUri);
+    await client.connect();
+    db = client.db('apartments_test');
+
+    // Create the test app with our database
+    app = await createTestApp(db);
+  });
+
+  afterAll(async () => {
+    if (db) {
+      await cleanupTestData(db);
+      await db.collection('settings').deleteMany({});
+    }
+    if (client) {
+      await client.close();
+    }
+  });
+
+  beforeEach(async () => {
+    // Clean up all test data
+    await cleanupTestData(db);
+    await db.collection('settings').deleteMany({});
+
+    // Create fresh test users with new ObjectIds for each test
+    testUsers = {
+      admin: createTestAdmin({
+        _id: new ObjectId(),
+        email: 'admin@example.com',
+        name: 'Test Admin'
+      }),
+      secondAdmin: createTestAdmin({
+        _id: new ObjectId(),
+        email: 'admin2@example.com',
+        name: 'Second Admin'
+      }),
+      regularUser: createTestUser({
+        _id: new ObjectId(),
+        email: 'user@example.com',
+        name: 'Test User'
+      }),
+      disabledUser: createTestUser({
+        _id: new ObjectId(),
+        email: 'disabled@example.com',
+        name: 'Disabled User',
+        isActive: false
+      })
+    };
+
+    // Insert primary admin and regular user
+    await db.collection('users').insertMany([
+      testUsers.admin,
+      testUsers.regularUser
+    ]);
+
+    // Insert default settings document
+    await db.collection('settings').insertOne({
+      _id: 'admin_settings',
+      activityRetentionDays: 90,
+      activityLogLevel: 'all',
+      updatedAt: new Date(),
+      updatedBy: null
+    });
+  });
+
+  // ============================================================
+  // API-4.1: GET /api/admin/settings - Get admin settings
+  // ============================================================
+  describe('API-4.1: GET /api/admin/settings', () => {
+    describe('Authorization', () => {
+      it('should return 401 when no authentication token is provided', async () => {
+        const response = await request(app)
+          .get('/api/admin/settings')
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 403 when user is not an admin', async () => {
+        const response = await authAs(testUsers.regularUser)
+          .get('/api/admin/settings')
+          .expect(403);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/admin/i);
+      });
+
+      it('should return 401 when admin user is disabled', async () => {
+        // Insert disabled admin
+        const disabledAdmin = createTestAdmin({
+          _id: new ObjectId(),
+          email: 'disabled-admin@example.com',
+          isActive: false
+        });
+        await db.collection('users').insertOne(disabledAdmin);
+
+        const response = await authAs(disabledAdmin)
+          .get('/api/admin/settings')
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 200 when user is an active admin', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(response.body).toHaveProperty('activityRetentionDays');
+        expect(response.body).toHaveProperty('activityLogLevel');
+      });
+    });
+
+    describe('Response format', () => {
+      it('should return activityRetentionDays as a number', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(typeof response.body.activityRetentionDays).toBe('number');
+      });
+
+      it('should return activityLogLevel as a string', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(typeof response.body.activityLogLevel).toBe('string');
+      });
+
+      it('should return default values when no settings document exists', async () => {
+        // Remove settings document
+        await db.collection('settings').deleteMany({});
+
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        // Should return defaults: 90 days retention, 'all' log level
+        expect(response.body.activityRetentionDays).toBe(90);
+        expect(response.body.activityLogLevel).toBe('all');
+      });
+
+      it('should return stored values when settings exist', async () => {
+        // Update settings to non-default values
+        await db.collection('settings').updateOne(
+          { _id: 'admin_settings' },
+          { $set: { activityRetentionDays: 180, activityLogLevel: 'navigation' } }
+        );
+
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(response.body.activityRetentionDays).toBe(180);
+        expect(response.body.activityLogLevel).toBe('navigation');
+      });
+    });
+  });
+
+  // ============================================================
+  // API-4.2: PATCH /api/admin/settings - Update settings
+  // ============================================================
+  describe('API-4.2: PATCH /api/admin/settings', () => {
+    describe('Authorization', () => {
+      it('should return 401 when no authentication token is provided', async () => {
+        const response = await request(app)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 403 when user is not an admin', async () => {
+        const response = await authAs(testUsers.regularUser)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(403);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/admin/i);
+      });
+
+      it('should return 200 when user is an active admin', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(200);
+
+        expect(response.body.data).toHaveProperty('settings');
+        expect(response.body.data).toHaveProperty('message');
+      });
+    });
+
+    describe('Validation - Retention period (API-4.2)', () => {
+      it('should return 400 for retention period below 30 days', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 29 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/30|365|range|invalid/i);
+      });
+
+      it('should return 400 for retention period above 365 days', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 366 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/30|365|range|invalid/i);
+      });
+
+      it('should accept retention period at minimum boundary (30 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 30 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(30);
+      });
+
+      it('should accept retention period at maximum boundary (365 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 365 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(365);
+      });
+
+      it('should accept valid retention period within range (180 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 180 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(180);
+      });
+
+      it('should return 400 for non-numeric retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 'ninety' })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 400 for negative retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: -30 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 400 for decimal retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 90.5 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+    });
+
+    describe('Successful update', () => {
+      it('should update retention period and return success message', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(120);
+        expect(response.body.data.message).toBe('Settings updated successfully');
+      });
+
+      it('should persist the updated settings in database', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 150 })
+          .expect(200);
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(settings.activityRetentionDays).toBe(150);
+      });
+
+      it('should update updatedAt timestamp', async () => {
+        const beforeUpdate = new Date();
+
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const afterUpdate = new Date();
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(new Date(settings.updatedAt).getTime()).toBeGreaterThanOrEqual(beforeUpdate.getTime());
+        expect(new Date(settings.updatedAt).getTime()).toBeLessThanOrEqual(afterUpdate.getTime());
+      });
+
+      it('should record which admin updated the settings', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(settings.updatedBy.toString()).toBe(testUsers.admin._id.toString());
+      });
+    });
+  });
+
+  // ============================================================
+  // API-4.3: TTL Index Update
+  // ============================================================
+  describe('API-4.3: TTL Index Update', () => {
+    it('should update TTL index when retention period changes to 60 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 60 })
+        .expect(200);
+
+      // Check the TTL index on user_activity collection
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 60 days = 60 * 24 * 60 * 60 = 5,184,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(60 * 24 * 60 * 60);
+    });
+
+    it('should update TTL index when retention period changes to 30 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 30 })
+        .expect(200);
+
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 30 days = 2,592,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(30 * 24 * 60 * 60);
+    });
+
+    it('should update TTL index when retention period changes to 365 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 365 })
+        .expect(200);
+
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 365 days = 31,536,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(365 * 24 * 60 * 60);
+    });
+
+    it('should correctly calculate TTL seconds for various retention periods', async () => {
+      const testCases = [
+        { days: 30, expectedSeconds: 2592000 },
+        { days: 90, expectedSeconds: 7776000 },
+        { days: 180, expectedSeconds: 15552000 },
+        { days: 365, expectedSeconds: 31536000 }
+      ];
+
+      for (const { days, expectedSeconds } of testCases) {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: days })
+          .expect(200);
+
+        const indexes = await db.collection('user_activity').indexes();
+        const ttlIndex = indexes.find(
+          idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+        );
+
+        expect(ttlIndex).toBeDefined();
+        expect(ttlIndex.expireAfterSeconds).toBe(expectedSeconds);
+      }
+    });
+  });
+
+  // ============================================================
+  // SEC-4.1: Admin Action Audit Logging
+  // ============================================================
+  describe('SEC-4.1: Admin Action Audit Logging', () => {
+    describe('ADMIN_UPDATE_SETTINGS', () => {
+      it('should log ADMIN_UPDATE_SETTINGS action when updating settings', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+      });
+
+      it('should record admin user ID in audit log', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+      });
+
+      it('should record new retention value in audit log metadata', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 150 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.metadata).toHaveProperty('activityRetentionDays', 150);
+      });
+
+      it('should include timestamp in audit log', async () => {
+        const beforeTime = new Date();
+
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const afterTime = new Date();
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.timestamp).toBeDefined();
+        expect(new Date(activity.timestamp).getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+        expect(new Date(activity.timestamp).getTime()).toBeLessThanOrEqual(afterTime.getTime());
+      });
+    });
+
+    describe('ADMIN_DISABLE_USER', () => {
+      it('should log ADMIN_DISABLE_USER action when disabling a user', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}`)
+          .send({ isActive: false })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_DISABLE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+      });
+    });
+
+    describe('ADMIN_ENABLE_USER', () => {
+      it('should log ADMIN_ENABLE_USER action when enabling a user', async () => {
+        // First disable the user
+        await db.collection('users').updateOne(
+          { _id: testUsers.regularUser._id },
+          { $set: { isActive: false } }
+        );
+
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}`)
+          .send({ isActive: true })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_ENABLE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+      });
+    });
+
+    describe('ADMIN_PROMOTE_USER', () => {
+      it('should log ADMIN_PROMOTE_USER action when promoting to admin', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}/role`)
+          .send({ role: 'admin' })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_PROMOTE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+        expect(activity.metadata.newRole).toBe('admin');
+      });
+    });
+
+    describe('ADMIN_DEMOTE_USER', () => {
+      beforeEach(async () => {
+        // Insert second admin for demotion tests
+        await db.collection('users').insertOne(testUsers.secondAdmin);
+      });
+
+      it('should log ADMIN_DEMOTE_USER action when demoting from admin', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+          .send({ role: 'user' })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_DEMOTE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.secondAdmin._id.toString());
+        expect(activity.metadata.newRole).toBe('user');
+      });
+    });
+  });
+
+  // ============================================================
+  // SEC-4.2: Last Admin Protection
+  // ============================================================
+  describe('SEC-4.2: Last Admin Protection', () => {
+    it('should return 400 when trying to demote the last admin', async () => {
+      // Ensure only one admin exists
+      await db.collection('users').deleteMany({
+        role: 'admin',
+        _id: { $ne: testUsers.admin._id }
+      });
+
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body).toHaveProperty('error');
+      expect(response.body.error).toMatch(/last admin|at least one admin|cannot demote/i);
+    });
+
+    it('should allow demoting an admin when other active admins exist', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Demote second admin (should succeed)
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+        .send({ role: 'user' })
+        .expect(200);
+
+      expect(response.body.data.user.role).toBe('user');
+    });
+
+    it('should not allow self-demotion even when other admins exist', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Try to demote self
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body).toHaveProperty('error');
+      expect(response.body.error).toMatch(/cannot demote yourself|self|own/i);
+    });
+
+    it('should count only active admins when checking for last admin', async () => {
+      // Insert a disabled admin (should not count)
+      const disabledAdmin = createTestAdmin({
+        _id: new ObjectId(),
+        email: 'disabled-admin@example.com',
+        isActive: false
+      });
+      await db.collection('users').insertOne(disabledAdmin);
+
+      // Try to demote self (should fail - only one active admin)
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body.error).toMatch(/last admin|at least one admin|cannot demote/i);
+    });
+
+    it('should verify admin count after demotion would leave at least one admin', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Verify we can demote the second admin
+      const response1 = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+        .send({ role: 'user' })
+        .expect(200);
+
+      expect(response1.body.data.user.role).toBe('user');
+
+      // Now the primary admin is the last one, cannot demote
+      const response2 = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response2.body.error).toMatch(/last admin|at least one admin/i);
+    });
+  });
+
+  // ============================================================
+  // SEC-4.3: Rate Limiting on Admin Endpoints
+  // SKIPPED: Rate limiting moved to Phase 5
+  // ============================================================
+  describe.skip('SEC-4.3: Rate Limiting', () => {
+    /**
+     * Helper to make multiple rapid requests
+     * @param {string} endpoint - API endpoint
+     * @param {number} count - Number of requests
+     * @param {string} method - HTTP method
+     * @param {Object} body - Request body for POST/PATCH
+     * @returns {Promise<Array>} Array of responses
+     */
+    async function makeRapidRequests(endpoint, count, method = 'get', body = null) {
+      const requests = [];
+      for (let i = 0; i < count; i++) {
+        let req = authAs(testUsers.admin)[method](endpoint);
+        if (body && (method === 'patch' || method === 'post')) {
+          req = req.send(body);
+        }
+        requests.push(req);
+      }
+      return Promise.all(requests);
+    }
+
+    describe('User list endpoint rate limiting (60 req/min)', () => {
+      it('should allow up to 60 requests per minute to user list', async () => {
+        const responses = await makeRapidRequests('/api/admin/users', 60);
+
+        // All 60 requests should succeed
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(60);
+      });
+
+      it('should return 429 when exceeding 60 requests per minute', async () => {
+        const responses = await makeRapidRequests('/api/admin/users', 65);
+
+        // At least some should be rate limited
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+
+        // Rate limited response should have appropriate error
+        if (rateLimited.length > 0) {
+          expect(rateLimited[0].body).toHaveProperty('error');
+          expect(rateLimited[0].body.error).toMatch(/rate limit|too many requests/i);
+        }
+      });
+    });
+
+    describe('User updates endpoint rate limiting (30 req/min)', () => {
+      it('should allow up to 30 requests per minute for user updates', async () => {
+        const responses = await makeRapidRequests(
+          `/api/admin/users/${testUsers.regularUser._id}`,
+          30,
+          'patch',
+          { isActive: true }
+        );
+
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(30);
+      });
+
+      it('should return 429 when exceeding 30 user update requests per minute', async () => {
+        const responses = await makeRapidRequests(
+          `/api/admin/users/${testUsers.regularUser._id}`,
+          35,
+          'patch',
+          { isActive: true }
+        );
+
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe('Stats endpoints rate limiting (30 req/min)', () => {
+      const statsEndpoints = [
+        '/api/admin/stats/overview',
+        '/api/admin/stats/active-users',
+        '/api/admin/stats/actions',
+        '/api/admin/stats/peak-times'
+      ];
+
+      it('should allow up to 30 requests per minute to stats overview', async () => {
+        const responses = await makeRapidRequests('/api/admin/stats/overview', 30);
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(30);
+      });
+
+      it('should return 429 when exceeding 30 stats requests per minute', async () => {
+        const responses = await makeRapidRequests('/api/admin/stats/overview', 35);
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+
+      it('should share rate limit across all stats endpoints', async () => {
+        // Make requests across different stats endpoints
+        const requests = [];
+        for (let i = 0; i < 40; i++) {
+          const endpoint = statsEndpoints[i % statsEndpoints.length];
+          requests.push(authAs(testUsers.admin).get(endpoint));
+        }
+
+        const responses = await Promise.all(requests);
+        const rateLimited = responses.filter(r => r.status === 429);
+
+        // After 30 total requests, should start rate limiting
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe('Rate limit headers', () => {
+      it('should include rate limit headers in response', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        // Check for common rate limit header variations
+        const hasRateLimitHeader =
+          response.headers['x-ratelimit-limit'] ||
+          response.headers['ratelimit-limit'] ||
+          response.headers['x-rate-limit-limit'];
+
+        expect(hasRateLimitHeader).toBeDefined();
+      });
+
+      it('should include remaining requests in headers', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        const hasRemainingHeader =
+          response.headers['x-ratelimit-remaining'] ||
+          response.headers['ratelimit-remaining'] ||
+          response.headers['x-rate-limit-remaining'];
+
+        expect(hasRemainingHeader).toBeDefined();
+      });
+
+      it('should include reset time in headers', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        const hasResetHeader =
+          response.headers['x-ratelimit-reset'] ||
+          response.headers['ratelimit-reset'] ||
+          response.headers['x-rate-limit-reset'];
+
+        expect(hasResetHeader).toBeDefined();
+      });
+    });
+  });
+});
+
+// ============================================================
+// Integration Test: Full Settings Update Workflow
+// ============================================================
+describe('Integration: Settings Update Workflow', () => {
+  let app;
+  let db;
+  let client;
+  let testAdmin;
+
+  beforeAll(async () => {
+    const mongoUri = process.env.MONGO_URI || 'mongodb://localhost:27017';
+    client = new MongoClient(mongoUri);
+    await client.connect();
+    db = client.db('apartments_test');
+    app = await createTestApp(db);
+  });
+
+  afterAll(async () => {
+    if (db) {
+      await cleanupTestData(db);
+      await db.collection('settings').deleteMany({});
+    }
+    if (client) {
+      await client.close();
+    }
+  });
+
+  beforeEach(async () => {
+    await cleanupTestData(db);
+    await db.collection('settings').deleteMany({});
+
+    testAdmin = createTestAdmin({
+      _id: new ObjectId(),
+      email: 'integration-admin@example.com'
+    });
+    await db.collection('users').insertOne(testAdmin);
+
+    await db.collection('settings').insertOne({
+      _id: 'admin_settings',
+      activityRetentionDays: 90,
+      activityLogLevel: 'all',
+      updatedAt: new Date(),
+      updatedBy: null
+    });
+  });
+
+  it('should complete full settings update workflow', async () => {
+    const authAdmin = (method, url) => {
+      const token = generateTestToken(testAdmin._id);
+      return request(app)[method](url).set('Cookie', `auth_token=${token}`);
+    };
+
+    // Step 1: Get current settings
+    const getResponse = await authAdmin('get', '/api/admin/settings')
+      .expect(200);
+
+    expect(getResponse.body.activityRetentionDays).toBe(90);
+
+    // Step 2: Update settings to new value
+    const updateResponse = await authAdmin('patch', '/api/admin/settings')
+      .send({ activityRetentionDays: 180 })
+      .expect(200);
+
+    expect(updateResponse.body.data.settings.activityRetentionDays).toBe(180);
+    expect(updateResponse.body.data.message).toBe('Settings updated successfully');
+
+    // Step 3: Verify settings persisted
+    const verifyResponse = await authAdmin('get', '/api/admin/settings')
+      .expect(200);
+
+    expect(verifyResponse.body.activityRetentionDays).toBe(180);
+
+    // Step 4: Verify audit log was created
+    const auditLog = await db.collection('user_activity').findOne({
+      action: 'ADMIN_UPDATE_SETTINGS'
+    });
+
+    expect(auditLog).toBeDefined();
+    expect(auditLog).not.toBeNull();
+    expect(auditLog.metadata.activityRetentionDays).toBe(180);
+    expect(auditLog.userId.toString()).toBe(testAdmin._id.toString());
+
+    // Step 5: Verify TTL index was updated
+    const indexes = await db.collection('user_activity').indexes();
+    const ttlIndex = indexes.find(idx => idx.name === 'timestamp_ttl');
+
+    expect(ttlIndex).toBeDefined();
+    expect(ttlIndex.expireAfterSeconds).toBe(180 * 24 * 60 * 60);
+
+    // Step 6: Verify database document was updated
+    const dbSettings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+    expect(dbSettings.activityRetentionDays).toBe(180);
+    expect(dbSettings.updatedBy.toString()).toBe(testAdmin._id.toString());
+  });
+
+  it('should handle multiple sequential settings updates', async () => {
+    const authAdmin = (method, url) => {
+      const token = generateTestToken(testAdmin._id);
+      return request(app)[method](url).set('Cookie', `auth_token=${token}`);
+    };
+
+    const retentionValues = [30, 60, 90, 180, 365];
+
+    for (const value of retentionValues) {
+      const response = await authAdmin('patch', '/api/admin/settings')
+        .send({ activityRetentionDays: value })
+        .expect(200);
+
+      expect(response.body.data.settings.activityRetentionDays).toBe(value);
+
+      // Verify TTL index after each update
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(idx => idx.name === 'timestamp_ttl');
+      expect(ttlIndex.expireAfterSeconds).toBe(value * 24 * 60 * 60);
+    }
+
+    // Should have 5 audit log entries
+    const auditLogCount = await db.collection('user_activity').countDocuments({
+      action: 'ADMIN_UPDATE_SETTINGS'
+    });
+    expect(auditLogCount).toBe(5);
+  });
+});

__tests__/scraper/dataConversion.test.js

@@ -0,0 +1,496 @@
+/**
+ * Tests for convertDataTypes() and its internal helper functions
+ *
+ * Covers:
+ * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
+ * - parseInteger: valid integers, null/empty/invalid
+ * - parsePositiveInteger: treats 0 as null (for area)
+ * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
+ * - parseFloat: "1.5" for half baths, null/empty/invalid
+ * - parseBoolean: "true"/"false"/"1"/"0", null/empty
+ * - trimString: trim whitespace, null for empty, null/undefined
+ * - convertDataTypes: full integration applying correct parser to each field
+ */
+
+const {
+  convertDataTypes,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString
+} = require('../../services/scraperService');
+
+// ---------------------------------------------------------------
+// 1. parsePrice
+// ---------------------------------------------------------------
+describe('parsePrice', () => {
+  it('should parse "$1,234" format to 1234', () => {
+    expect(parsePrice('$1,234')).toBe(1234);
+  });
+
+  it('should parse plain numeric string', () => {
+    expect(parsePrice('1450')).toBe(1450);
+  });
+
+  it('should parse "$2,500.00" stripping non-numeric chars', () => {
+    expect(parsePrice('$2,500.00')).toBe(2500);
+  });
+
+  it('should return null for "Call for pricing"', () => {
+    expect(parsePrice('Call for pricing')).toBeNull();
+  });
+
+  it('should return null for "Contact us"', () => {
+    expect(parsePrice('Contact us')).toBeNull();
+  });
+
+  it('should return null for "Please inquire"', () => {
+    expect(parsePrice('Please inquire')).toBeNull();
+  });
+
+  it('should return null for negative values', () => {
+    // After stripping non-numeric chars, "-1500" becomes "1500"
+    // but if someone passes a numeric -1500 directly, check that too
+    expect(parsePrice('-')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(parsePrice(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePrice('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parsePrice(undefined)).toBeNull();
+  });
+
+  it('should return null for completely non-numeric string', () => {
+    expect(parsePrice('abc')).toBeNull();
+  });
+
+  it('should handle price of 0', () => {
+    expect(parsePrice('0')).toBe(0);
+  });
+
+  it('should handle large prices', () => {
+    expect(parsePrice('$15,500')).toBe(15500);
+  });
+});
+
+// ---------------------------------------------------------------
+// 2. parseInteger
+// ---------------------------------------------------------------
+describe('parseInteger', () => {
+  it('should parse valid integer string', () => {
+    expect(parseInteger('42')).toBe(42);
+  });
+
+  it('should parse "0" to 0', () => {
+    expect(parseInteger('0')).toBe(0);
+  });
+
+  it('should parse negative integer string', () => {
+    expect(parseInteger('-5')).toBe(-5);
+  });
+
+  it('should return null for null', () => {
+    expect(parseInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseInteger('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseInteger(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseInteger('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseInteger('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseInteger('Infinity')).toBeNull();
+  });
+
+  it('should truncate float strings to integer', () => {
+    expect(parseInteger('3.7')).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------
+// 3. parsePositiveInteger
+// ---------------------------------------------------------------
+describe('parsePositiveInteger', () => {
+  it('should parse positive integer normally', () => {
+    expect(parsePositiveInteger('750')).toBe(750);
+  });
+
+  it('should treat 0 as null (for area field)', () => {
+    expect(parsePositiveInteger('0')).toBeNull();
+  });
+
+  it('should return null for null input', () => {
+    expect(parsePositiveInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePositiveInteger('')).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parsePositiveInteger('abc')).toBeNull();
+  });
+
+  it('should handle negative values (parsed as negative int, not positive)', () => {
+    expect(parsePositiveInteger('-5')).toBe(-5);
+  });
+});
+
+// ---------------------------------------------------------------
+// 4. parseIntegerOrString
+// ---------------------------------------------------------------
+describe('parseIntegerOrString', () => {
+  it('should return integer when value is cleanly parseable', () => {
+    expect(parseIntegerOrString('5001')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric value', () => {
+    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
+  });
+
+  it('should return trimmed string when value has non-numeric suffix', () => {
+    expect(parseIntegerOrString('123abc')).toBe('123abc');
+  });
+
+  it('should return null for null', () => {
+    expect(parseIntegerOrString(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseIntegerOrString('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseIntegerOrString(undefined)).toBeNull();
+  });
+
+  it('should trim whitespace before checking', () => {
+    expect(parseIntegerOrString('  5001  ')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric with whitespace', () => {
+    expect(parseIntegerOrString('  abc  ')).toBe('abc');
+  });
+});
+
+// ---------------------------------------------------------------
+// 5. parseFloatValue
+// ---------------------------------------------------------------
+describe('parseFloatValue', () => {
+  it('should parse "1.5" for half baths', () => {
+    expect(parseFloatValue('1.5')).toBe(1.5);
+  });
+
+  it('should parse "2.0" to 2', () => {
+    expect(parseFloatValue('2.0')).toBe(2.0);
+  });
+
+  it('should parse integer string as float', () => {
+    expect(parseFloatValue('3')).toBe(3);
+  });
+
+  it('should return null for null', () => {
+    expect(parseFloatValue(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseFloatValue('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseFloatValue(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseFloatValue('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseFloatValue('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseFloatValue('Infinity')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 6. parseBoolean
+// ---------------------------------------------------------------
+describe('parseBoolean', () => {
+  it('should parse "true" to true', () => {
+    expect(parseBoolean('true')).toBe(true);
+  });
+
+  it('should parse "false" to false', () => {
+    expect(parseBoolean('false')).toBe(false);
+  });
+
+  it('should parse "1" to true', () => {
+    expect(parseBoolean('1')).toBe(true);
+  });
+
+  it('should parse "0" to false', () => {
+    expect(parseBoolean('0')).toBe(false);
+  });
+
+  it('should parse "TRUE" (case insensitive) to true', () => {
+    expect(parseBoolean('TRUE')).toBe(true);
+  });
+
+  it('should parse "False" (case insensitive) to false', () => {
+    expect(parseBoolean('False')).toBe(false);
+  });
+
+  it('should pass through boolean true', () => {
+    expect(parseBoolean(true)).toBe(true);
+  });
+
+  it('should pass through boolean false', () => {
+    expect(parseBoolean(false)).toBe(false);
+  });
+
+  it('should return null for null', () => {
+    expect(parseBoolean(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseBoolean('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseBoolean(undefined)).toBeNull();
+  });
+
+  it('should return null for unrecognized string', () => {
+    expect(parseBoolean('yes')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 7. trimString
+// ---------------------------------------------------------------
+describe('trimString', () => {
+  it('should trim leading and trailing whitespace', () => {
+    expect(trimString('  hello  ')).toBe('hello');
+  });
+
+  it('should return the string as-is if no whitespace', () => {
+    expect(trimString('hello')).toBe('hello');
+  });
+
+  it('should return null for empty string', () => {
+    expect(trimString('')).toBeNull();
+  });
+
+  it('should return null for whitespace-only string', () => {
+    expect(trimString('   ')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(trimString(null)).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(trimString(undefined)).toBeNull();
+  });
+
+  it('should convert non-string to string then trim', () => {
+    expect(trimString(123)).toBe('123');
+  });
+});
+
+// ---------------------------------------------------------------
+// 8. convertDataTypes - Full integration test
+// ---------------------------------------------------------------
+describe('convertDataTypes', () => {
+  it('should apply correct parser to each field', () => {
+    const rawUnit = {
+      id: '1001',
+      unit_code: 'CCT-101',
+      unit_id: '5001',
+      floor: '1',
+      area: '750',
+      bed_count: '1',
+      bath_count: '1.5',
+      price: '1450',
+      available: 'true',
+      unavailable: 'false',
+      soonest: '2026-03-01',
+      date_available: '20260301',
+      plan_id: '201',
+      plan_name: 'Alpine',
+      obj_type: 'unit',
+      community: 'Country Club Towers',
+      asset: '100',
+      href: '/units/CCT-101',
+      inventory_href: '/inventory/CCT-101',
+      image_url: 'https://example.com/images/unit-101.jpg',
+      specials_content: 'First month free!'
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    // Integer fields
+    expect(converted.id).toBe(1001);
+    expect(converted.floor).toBe(1);
+    expect(converted.bed_count).toBe(1);
+    expect(converted.plan_id).toBe(201);
+    expect(converted.asset).toBe(100);
+    expect(converted.date_available).toBe(20260301);
+
+    // parseIntegerOrString
+    expect(converted.unit_id).toBe(5001);
+
+    // parsePositiveInteger (area)
+    expect(converted.area).toBe(750);
+
+    // Float field (bath_count)
+    expect(converted.bath_count).toBe(1.5);
+
+    // Price field
+    expect(converted.price).toBe(1450);
+
+    // Boolean fields
+    expect(converted.available).toBe(true);
+    expect(converted.unavailable).toBe(false);
+
+    // String fields (trimmed)
+    expect(converted.unit_code).toBe('CCT-101');
+    expect(converted.plan_name).toBe('Alpine');
+    expect(converted.soonest).toBe('2026-03-01');
+    expect(converted.obj_type).toBe('unit');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.href).toBe('/units/CCT-101');
+    expect(converted.inventory_href).toBe('/inventory/CCT-101');
+    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
+    expect(converted.specials_content).toBe('First month free!');
+  });
+
+  it('should preserve string fields without converting them', () => {
+    const rawUnit = {
+      unit_code: 'CCT-205',
+      plan_name: '  Birch  ',
+      soonest: 'Now',
+      obj_type: 'unit',
+      community: '  Country Club Towers  ',
+      href: '/units/CCT-205',
+      inventory_href: '/inventory/CCT-205',
+      image_url: 'https://example.com/img.jpg',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.unit_code).toBe('CCT-205');
+    expect(converted.plan_name).toBe('Birch');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle a unit with all null/undefined/empty values', () => {
+    const rawUnit = {
+      id: null,
+      unit_code: undefined,
+      unit_id: '',
+      floor: null,
+      area: '',
+      bed_count: undefined,
+      bath_count: null,
+      price: '',
+      available: null,
+      unavailable: undefined,
+      soonest: '',
+      date_available: null,
+      plan_id: undefined,
+      plan_name: '',
+      obj_type: null,
+      community: undefined,
+      asset: '',
+      href: null,
+      inventory_href: undefined,
+      image_url: '',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.id).toBeNull();
+    expect(converted.unit_code).toBeNull();
+    expect(converted.unit_id).toBeNull();
+    expect(converted.floor).toBeNull();
+    expect(converted.area).toBeNull();
+    expect(converted.bed_count).toBeNull();
+    expect(converted.bath_count).toBeNull();
+    expect(converted.price).toBeNull();
+    expect(converted.available).toBeNull();
+    expect(converted.unavailable).toBeNull();
+    expect(converted.soonest).toBeNull();
+    expect(converted.date_available).toBeNull();
+    expect(converted.plan_id).toBeNull();
+    expect(converted.plan_name).toBeNull();
+    expect(converted.obj_type).toBeNull();
+    expect(converted.community).toBeNull();
+    expect(converted.asset).toBeNull();
+    expect(converted.href).toBeNull();
+    expect(converted.inventory_href).toBeNull();
+    expect(converted.image_url).toBeNull();
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle area=0 as null (parsePositiveInteger)', () => {
+    const rawUnit = { area: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.area).toBeNull();
+  });
+
+  it('should handle bed_count=0 as valid (studio)', () => {
+    const rawUnit = { bed_count: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bed_count).toBe(0);
+  });
+
+  it('should handle "Call for pricing" in price field', () => {
+    const rawUnit = { price: 'Call for pricing' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBeNull();
+  });
+
+  it('should handle "$1,234" price format', () => {
+    const rawUnit = { price: '$1,234' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBe(1234);
+  });
+
+  it('should handle unit_id as string identifier', () => {
+    const rawUnit = { unit_id: 'UNIT-ABC' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.unit_id).toBe('UNIT-ABC');
+  });
+
+  it('should handle bath_count with half baths', () => {
+    const rawUnit = { bath_count: '1.5' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bath_count).toBe(1.5);
+  });
+});

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

__tests__/scraper/fixtures/sample-listing-call.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Call for Pricing</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9001"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="750"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="P-101"
+    data-spaces-unit-id="9001"
+    data-spaces-unit-floor="1"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="500"
+    data-spaces-sort-plan-name="Penthouse A"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9001"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
+    aria-label="Unit P-101">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-101</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9002"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="900"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="P-102"
+    data-spaces-unit-id="9002"
+    data-spaces-unit-floor="2"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="501"
+    data-spaces-sort-plan-name="Penthouse B"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9002"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
+    aria-label="Unit P-102">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-102</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing-empty.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - No Units</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <!-- Empty - no units available -->
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Apartments</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154842"
+    data-spaces-soonest="2025-10-11"
+    data-spaces-sort-date="1760140800"
+    data-spaces-sort-price="5230"
+    data-spaces-sort-area="1210"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W2707"
+    data-spaces-unit-id="154842"
+    data-spaces-unit-floor="2755"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="10270"
+    data-spaces-sort-plan-name="Pyramid Peak - Terrace"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154842"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
+    aria-label="Unit W2707">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">5230</span>
+      <span class="spaces-unit__unit">W2707</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154451"
+    data-spaces-soonest="2026-03-08"
+    data-spaces-sort-date="1772928000"
+    data-spaces-sort-price="2767"
+    data-spaces-sort-area="806"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="E0901"
+    data-spaces-unit-id="154451"
+    data-spaces-unit-floor="2738"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8016"
+    data-spaces-sort-plan-name="Sunshine Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154451"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
+    aria-label="Unit E0901">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2767</span>
+      <span class="spaces-unit__unit">E0901</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154814"
+    data-spaces-soonest="2025-12-18"
+    data-spaces-sort-date="1766016000"
+    data-spaces-sort-price="2255"
+    data-spaces-sort-area="623"
+    data-spaces-sort-bed="0"
+    data-spaces-unit="W2603"
+    data-spaces-unit-id="154814"
+    data-spaces-unit-floor="2754"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8018"
+    data-spaces-sort-plan-name="Little Bear Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154814"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
+    aria-label="Unit W2603">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2255</span>
+      <span class="spaces-unit__unit">W2603</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154428"
+    data-spaces-soonest="2026-02-22"
+    data-spaces-sort-date="1771718400"
+    data-spaces-sort-price="4250"
+    data-spaces-sort-area="1152"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W0802"
+    data-spaces-unit-id="154428"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8017"
+    data-spaces-sort-plan-name="Mt. Princeton"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154428"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
+    aria-label="Unit W0802">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4250</span>
+      <span class="spaces-unit__unit">W0802</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154778"
+    data-spaces-soonest="2025-12-31"
+    data-spaces-sort-date="1767139200"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="764"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2405"
+    data-spaces-unit-id="154778"
+    data-spaces-unit-floor="2752"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8020"
+    data-spaces-sort-plan-name="Grays Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154778"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
+    aria-label="Unit W2405">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W2405</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154442"
+    data-spaces-soonest="2025-02-01"
+    data-spaces-sort-date="1738368000"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="802"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W0809"
+    data-spaces-unit-id="154442"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8022"
+    data-spaces-sort-plan-name="Pikes Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154442"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
+    aria-label="Unit W0809">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W0809</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154764"
+    data-spaces-soonest="2026-02-17"
+    data-spaces-sort-date="1771286400"
+    data-spaces-sort-price="2435"
+    data-spaces-sort-area="715"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2308"
+    data-spaces-unit-id="154764"
+    data-spaces-unit-floor="2751"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8023"
+    data-spaces-sort-plan-name="Longs Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154764"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
+    aria-label="Unit W2308">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2435</span>
+      <span class="spaces-unit__unit">W2308</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154612"
+    data-spaces-soonest="2026-03-24"
+    data-spaces-sort-date="1774310400"
+    data-spaces-sort-price="2683"
+    data-spaces-sort-area="797"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W1610"
+    data-spaces-unit-id="154612"
+    data-spaces-unit-floor="2744"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8024"
+    data-spaces-sort-plan-name="Torreys Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154612"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
+    aria-label="Unit W1610">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2683</span>
+      <span class="spaces-unit__unit">W1610</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154495"
+    data-spaces-soonest="2025-11-07"
+    data-spaces-sort-date="1762473600"
+    data-spaces-sort-price="4470"
+    data-spaces-sort-area="1230"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E1011"
+    data-spaces-unit-id="154495"
+    data-spaces-unit-floor="2739"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8025"
+    data-spaces-sort-plan-name="Maroon Peak"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154495"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
+    aria-label="Unit E1011">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4470</span>
+      <span class="spaces-unit__unit">E1011</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="154919"
+    data-spaces-soonest="2025-10-27"
+    data-spaces-sort-date="1761523200"
+    data-spaces-sort-price="8581"
+    data-spaces-sort-area="1532"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E3205"
+    data-spaces-unit-id="154919"
+    data-spaces-unit-floor="2760"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8037"
+    data-spaces-sort-plan-name="Mt. Antero"
+    data-spaces-sort-bath="2.5"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154919"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
+    aria-label="Unit E3205">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">8581</span>
+      <span class="spaces-unit__unit">E3205</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/insertPrices.test.js

@@ -0,0 +1,582 @@
+/**
+ * Tests for insertPrices() bulk operation
+ *
+ * Covers:
+ * - Only units with non-null prices are inserted
+ * - bulkWrite uses updateOne with upsert for idempotency
+ * - Filter uses unit_code + date_checked composite key
+ * - Price record includes unit_code, date_checked, price, last_updated, data_source
+ * - Empty price records logs warning and returns early
+ * - Error handling when bulkWrite fails
+ * - Logging of inserted and updated counts
+ * - Re-running on same day updates existing records (idempotent)
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require insertPrices after implementation
+let insertPrices;
+
+let mongoServer;
+let client;
+let db;
+
+const PRICES_COLLECTION = 'unit_prices_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ insertPrices } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the prices collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('insertPrices', () => {
+  const TODAY = '2026-02-05';
+
+  // ---------------------------------------------------------------
+  // 1. Only units with non-null prices are inserted
+  // ---------------------------------------------------------------
+  describe('filtering units with null prices', () => {
+    it('should only insert records for units with non-null prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200 },
+        { unit_code: 'A102', price: null },
+        { unit_code: 'A103', price: 1500 },
+        { unit_code: 'A104', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'A103']);
+    });
+
+    it('should skip units with price of null and not create any record for them', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'B101', price: null },
+        { unit_code: 'B102', price: null }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(0);
+    });
+
+    it('should handle a mix of priced and null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'C101', price: 950 },
+        { unit_code: 'C102', price: null },
+        { unit_code: 'C103', price: 1100 },
+        { unit_code: 'C104', price: null },
+        { unit_code: 'C105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. bulkWrite uses updateOne with upsert for idempotency
+  // ---------------------------------------------------------------
+  describe('bulkWrite with upsert operations', () => {
+    it('should create updateOne operations for each priced unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'D101', price: 1200 },
+        { unit_code: 'D102', price: 1500 }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.insertedCount).toBeGreaterThan(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+    });
+
+    it('should use upsert so re-running does not create duplicates', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'D201', price: 1200 }];
+
+      // First run
+      await insertPrices(db, units, TODAY, logger);
+
+      // Second run on same day
+      await insertPrices(db, units, TODAY, logger);
+
+      // Should still have only 1 document (not 2)
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Filter uses unit_code + date_checked composite key
+  // ---------------------------------------------------------------
+  describe('composite key: unit_code + date_checked', () => {
+    it('should use unit_code and date_checked as the filter for upsert', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E101', price: 1200 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'E101',
+        date_checked: TODAY
+      });
+
+      expect(doc).not.toBeNull();
+      expect(doc.unit_code).toBe('E101');
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should create separate records for the same unit on different days', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E201', price: 1200 }];
+
+      await insertPrices(db, units, '2026-02-04', logger);
+      await insertPrices(db, units, '2026-02-05', logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ unit_code: 'E201' })
+        .sort({ date_checked: 1 })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+      expect(docs[0].date_checked).toBe('2026-02-04');
+      expect(docs[1].date_checked).toBe('2026-02-05');
+    });
+
+    it('should create separate records for different units on the same day', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'E301', price: 1000 },
+        { unit_code: 'E302', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ date_checked: TODAY })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Price record includes required fields
+  // ---------------------------------------------------------------
+  describe('price record fields', () => {
+    it('should include unit_code in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.unit_code).toBe('F101');
+    });
+
+    it('should include date_checked in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F201', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should include price as a number in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F301', price: 1550 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.price).toBe(1550);
+      expect(typeof doc.price).toBe('number');
+    });
+
+    it('should include last_updated as an ISO timestamp string', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'F401', price: 1600 }];
+      await insertPrices(db, units, TODAY, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.last_updated).toBeDefined();
+      expect(typeof doc.last_updated).toBe('string');
+      expect(doc.last_updated >= beforeTime).toBe(true);
+      expect(doc.last_updated <= afterTime).toBe(true);
+    });
+
+    it('should include data_source set to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F501', price: 1700 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should include all five required fields in every price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F601', price: 1800 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc).toHaveProperty('unit_code');
+      expect(doc).toHaveProperty('date_checked');
+      expect(doc).toHaveProperty('price');
+      expect(doc).toHaveProperty('last_updated');
+      expect(doc).toHaveProperty('data_source');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty price records logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty price records', () => {
+    it('should log a warning when no units have prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'G101', price: null },
+        { unit_code: 'G102', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should return insertedCount of 0 for all null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'G201', price: null }];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+    });
+
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should not perform any database writes for empty/null-priced arrays', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H101', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H201', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await insertPrices(mockDb, units, TODAY, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to insert prices',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H301', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Logging of inserted and updated counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log inserted and updated counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I101', price: 1000 },
+        { unit_code: 'I102', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: expect.any(Number),
+          updated: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I201', price: 1000 },
+        { unit_code: 'I202', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // For new inserts: inserted=2, updated=0
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 2,
+          updated: 0
+        })
+      );
+    });
+
+    it('should log correct counts when updating existing records', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'I301', price: 1000 }];
+
+      // First insert
+      await insertPrices(db, units, TODAY, logger);
+
+      // Reset mock
+      logger.info.mockClear();
+
+      // Second run updates existing
+      const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
+      await insertPrices(db, updatedUnits, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 0,
+          updated: 1
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Re-running on same day updates existing records (idempotent)
+  // ---------------------------------------------------------------
+  describe('idempotent re-runs on same day', () => {
+    it('should update the price when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run with original price
+      await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
+
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc1.price).toBe(1200);
+
+      // Re-run with updated price (same day)
+      await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
+
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc2.price).toBe(1250);
+
+      // Should still have only 1 record
+      const count = await db.collection(PRICES_COLLECTION).countDocuments({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(count).toBe(1);
+    });
+
+    it('should update last_updated when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run
+      await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const firstUpdated = doc1.last_updated;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Re-run on same day
+      await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const secondUpdated = doc2.last_updated;
+
+      expect(secondUpdated > firstUpdated).toBe(true);
+    });
+
+    it('should return combined insert/update count reflecting the operation', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'J301', price: 1000 },
+        { unit_code: 'J302', price: 1100 }
+      ];
+
+      // First run - inserts
+      const result1 = await insertPrices(db, units, TODAY, logger);
+      expect(result1.insertedCount).toBe(2);
+
+      // Re-run - updates
+      const result2 = await insertPrices(db, units, TODAY, logger);
+      expect(result2.insertedCount).toBe(2); // combined upserted + modified
+
+      // Still only 2 total records
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(2);
+    });
+
+    it('should handle a mix of new and existing records on re-run', async () => {
+      const logger = createMockLogger();
+
+      // First run with unit A
+      await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
+
+      // Re-run with unit A (existing) and unit B (new)
+      const mixedUnits = [
+        { unit_code: 'J401', price: 1050 },
+        { unit_code: 'J402', price: 1200 }
+      ];
+      await insertPrices(db, mixedUnits, TODAY, logger);
+
+      // Should have 2 total records
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      // Verify updated price
+      const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
+      expect(unitA.price).toBe(1050);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Uses ordered: false for performance
+  // ---------------------------------------------------------------
+  describe('ordered: false for performance', () => {
+    it('should successfully process all priced units in a batch', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'K101', price: 900 },
+        { unit_code: 'K102', price: 1000 },
+        { unit_code: 'K103', price: 1100 },
+        { unit_code: 'K104', price: 1200 },
+        { unit_code: 'K105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Uses correct collection
+  // ---------------------------------------------------------------
+  describe('collection usage', () => {
+    it('should write to the configured PRICES collection', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'L101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // Verify data is in the correct collection
+      const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
+      expect(doc).not.toBeNull();
+    });
+  });
+});

__tests__/scraper/markStaleUnits.test.js

@@ -0,0 +1,449 @@
+/**
+ * Tests for markStaleUnits() availability tracking
+ *
+ * Covers:
+ * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
+ * - $set updates available to false and sets marked_stale_date
+ * - Units in currentCodes are NOT marked stale
+ * - Already unavailable units are not modified
+ * - Error handling when updateMany fails
+ * - Logging of modified count
+ * - Empty currentUnitCodes handles gracefully
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require markStaleUnits after implementation
+let markStaleUnits;
+
+let mongoServer;
+let client;
+let db;
+
+const UNITS_COLLECTION = 'units_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ markStaleUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('markStaleUnits', () => {
+  // ---------------------------------------------------------------
+  // 1. updateMany is called with correct filter
+  //    (unit_code $nin currentCodes AND available: true)
+  // ---------------------------------------------------------------
+  describe('filter criteria', () => {
+    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
+      const logger = createMockLogger();
+
+      // Seed units in the database
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'A101', available: true, price: 1200 },
+        { unit_code: 'A102', available: true, price: 1300 },
+        { unit_code: 'B201', available: true, price: 1500 }
+      ]);
+
+      // Current scrape found A101 and A102, but NOT B201
+      const currentCodes = new Set(['A101', 'A102']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // B201 should be marked stale (it was available but not in current scrape)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify B201 was updated
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
+      expect(staleUnit.available).toBe(false);
+      expect(staleUnit.marked_stale_date).toBe(date);
+    });
+
+    it('should only affect units NOT in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'X1', available: true },
+        { unit_code: 'X2', available: true },
+        { unit_code: 'X3', available: true },
+        { unit_code: 'X4', available: true }
+      ]);
+
+      // Current scrape found X1 and X3
+      const currentCodes = new Set(['X1', 'X3']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // X1 and X3 should still be available
+      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
+      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
+      expect(x1.available).toBe(true);
+      expect(x3.available).toBe(true);
+
+      // X2 and X4 should be marked stale
+      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
+      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
+      expect(x2.available).toBe(false);
+      expect(x4.available).toBe(false);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. $set updates available to false and sets marked_stale_date
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set available to false for stale units', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'S100', available: true, price: 1000 },
+        { unit_code: 'S200', available: true, price: 1100 }
+      ]);
+
+      const currentCodes = new Set(['S100']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
+      expect(staleUnit.available).toBe(false);
+    });
+
+    it('should set marked_stale_date to the provided date', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'D100', available: true }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
+      expect(doc.marked_stale_date).toBe('2026-02-05');
+    });
+
+    it('should update both available and marked_stale_date in the same operation', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'C100', available: true, price: 900 }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-01-15';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
+      expect(doc.available).toBe(false);
+      expect(doc.marked_stale_date).toBe('2026-01-15');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Units in currentCodes are NOT marked stale
+  // ---------------------------------------------------------------
+  describe('units in currentCodes not affected', () => {
+    it('should not modify units that are in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'KEEP-1', available: true, price: 1200 },
+        { unit_code: 'KEEP-2', available: true, price: 1300 },
+        { unit_code: 'STALE-1', available: true, price: 1400 }
+      ]);
+
+      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // Kept units should retain their original state
+      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
+      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
+
+      expect(keep1.available).toBe(true);
+      expect(keep1.marked_stale_date).toBeUndefined();
+      expect(keep2.available).toBe(true);
+      expect(keep2.marked_stale_date).toBeUndefined();
+    });
+
+    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ALL-1', available: true },
+        { unit_code: 'ALL-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['ALL-1', 'ALL-2']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Already unavailable units are not modified
+  // ---------------------------------------------------------------
+  describe('already unavailable units', () => {
+    it('should not modify units that are already unavailable (available: false)', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
+        { unit_code: 'UNAVAIL-2', available: false },
+        { unit_code: 'AVAIL-1', available: true }
+      ]);
+
+      // None of these units are in the current scrape
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify the already-unavailable unit's date was NOT changed
+      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
+      expect(unavail1.marked_stale_date).toBe('2026-01-01');
+    });
+
+    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'U1', available: false },
+        { unit_code: 'U2', available: false }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Error handling when updateMany fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      try {
+        await markStaleUnits(mockDb, currentCodes, date, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to mark stale units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging of modified count
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of stale units marked', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'LOG-1', available: true },
+        { unit_code: 'LOG-2', available: true },
+        { unit_code: 'LOG-3', available: true }
+      ]);
+
+      const currentCodes = new Set(['LOG-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 2 })
+      );
+    });
+
+    it('should log count of 0 when no units are marked stale', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ZERO-1', available: true }
+      ]);
+
+      const currentCodes = new Set(['ZERO-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Empty currentUnitCodes handles gracefully
+  // ---------------------------------------------------------------
+  describe('empty currentUnitCodes', () => {
+    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'EMPTY-1', available: true },
+        { unit_code: 'EMPTY-2', available: true },
+        { unit_code: 'EMPTY-3', available: true }
+      ]);
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(3);
+
+      // All should be marked unavailable
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      for (const doc of docs) {
+        expect(doc.available).toBe(false);
+        expect(doc.marked_stale_date).toBe('2026-02-05');
+      }
+    });
+
+    it('should not throw when currentUnitCodes is an empty Set', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      // Should not throw even with empty collection and empty set
+      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
+    });
+
+    it('should handle empty database with empty currentUnitCodes', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the MongoDB updateMany result', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'RET-1', available: true },
+        { unit_code: 'RET-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['RET-1']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // updateMany result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('acknowledged');
+      expect(result.acknowledged).toBe(true);
+    });
+  });
+});

__tests__/scraper/parseUnits.test.js

@@ -0,0 +1,431 @@
+/**
+ * Tests for parseUnits() - HTML parsing with cheerio
+ *
+ * Covers:
+ * - Extraction of all data-spaces-* attributes
+ * - Image URL extraction from nested img elements
+ * - Container detection (section.spaces__tab-unit)
+ * - Empty / missing container handling
+ * - Deduplication by unit_code
+ * - Logging behavior (info, warn, error)
+ * - Missing attribute handling
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseUnits } = require('../../services/scraperService');
+
+const fixturesDir = path.join(__dirname, 'fixtures');
+
+// Helper: create a mock logger
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+describe('parseUnits', () => {
+  let sampleHtml;
+  let mockLogger;
+
+  beforeAll(() => {
+    sampleHtml = fs.readFileSync(
+      path.join(fixturesDir, 'sample-listing.html'),
+      'utf-8'
+    );
+  });
+
+  beforeEach(() => {
+    mockLogger = createMockLogger();
+  });
+
+  // ---------------------------------------------------------------
+  // 1. Container detection
+  // ---------------------------------------------------------------
+  describe('container detection', () => {
+    it('should find section.spaces__tab-unit container in valid HTML', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+
+      // Should not log an error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+      expect(units.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array when section container is missing', () => {
+      const html = '<html><body><div>No units here</div></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+
+    it('should return empty array for empty HTML string', () => {
+      const units = parseUnits('', mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    it('should return empty array when container exists but has no articles', () => {
+      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container found, no error about container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Extraction of all data-spaces-* attributes
+  // ---------------------------------------------------------------
+  describe('attribute extraction', () => {
+    it('should extract all expected attributes from a fully populated article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'W2707');
+
+      expect(unit1).toBeDefined();
+
+      // Core identifiers
+      expect(unit1.id).toBe('154842');
+      expect(unit1.unit_code).toBe('W2707');
+      expect(unit1.unit_id).toBe('154842');
+
+      // Physical attributes
+      expect(unit1.floor).toBe('2755');
+      expect(unit1.area).toBe('1210');
+      expect(unit1.bed_count).toBe('2');
+      expect(unit1.bath_count).toBe('2');
+
+      // Pricing
+      expect(unit1.price).toBe('5230');
+
+      // Availability
+      expect(unit1.available).toBe('true');
+      expect(unit1.unavailable).toBe('false');
+      expect(unit1.soonest).toBe('2025-10-11');
+      expect(unit1.date_available).toBe('1760140800');
+
+      // Plan information
+      expect(unit1.plan_id).toBe('10270');
+      expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
+
+      // Property information
+      expect(unit1.obj_type).toBe('unit');
+      expect(unit1.community).toBe('Country Club Towers');
+      expect(unit1.asset).toBe('420');
+
+      // URLs
+      expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
+      expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
+
+      // Specials
+      expect(unit1.specials_content).toBe('');
+    });
+
+    it('should extract correct count of units from the fixture', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      expect(units).toHaveLength(10);
+    });
+
+    it('should return raw string values without type conversion', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'W2707');
+
+      // All values should be strings (raw extraction, no conversion)
+      expect(typeof unit.id).toBe('string');
+      expect(typeof unit.price).toBe('string');
+      expect(typeof unit.floor).toBe('string');
+      expect(typeof unit.available).toBe('string');
+      expect(typeof unit.bed_count).toBe('string');
+      expect(typeof unit.bath_count).toBe('string');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Image URL extraction
+  // ---------------------------------------------------------------
+  describe('image URL extraction', () => {
+    it('should extract image_url from img src attribute', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
+            <img src="https://example.com/images/unit-1.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
+    });
+
+    it('should fall back to data-src when src is not present', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
+            <img data-src="https://example.com/images/unit-2.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
+    });
+
+    it('should not set image_url when no img element exists', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
+            <div>No image here</div>
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Missing attributes
+  // ---------------------------------------------------------------
+  describe('missing attributes', () => {
+    it('should return undefined for attributes not present on the article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit).toBeDefined();
+      expect(unit.floor).toBeUndefined();
+      expect(unit.area).toBeUndefined();
+      expect(unit.bed_count).toBeUndefined();
+      expect(unit.bath_count).toBeUndefined();
+      expect(unit.soonest).toBeUndefined();
+      expect(unit.date_available).toBeUndefined();
+      expect(unit.plan_id).toBeUndefined();
+      expect(unit.plan_name).toBeUndefined();
+      expect(unit.href).toBeUndefined();
+      expect(unit.inventory_href).toBeUndefined();
+      expect(unit.specials_content).toBeUndefined();
+    });
+
+    it('should still extract the attributes that are present on a sparse article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.id).toBe('1005');
+      expect(unit.unit_code).toBe('SPARSE-1');
+      expect(unit.unit_id).toBe('5005');
+      expect(unit.price).toBe('1300');
+      expect(unit.available).toBe('true');
+      expect(unit.obj_type).toBe('unit');
+      expect(unit.community).toBe('Country Club Towers');
+      expect(unit.asset).toBe('100');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Deduplication
+  // ---------------------------------------------------------------
+  describe('deduplication', () => {
+    it('should deduplicate units with the same unit_code', () => {
+      // Build HTML with duplicate articles
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+      const codes = units.map(u => u.unit_code);
+
+      // Should have 2 unique unit codes
+      expect(units).toHaveLength(2);
+      expect(new Set(codes).size).toBe(codes.length);
+    });
+
+    it('should keep the first occurrence when deduplicating', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // First occurrence (id=1, price=1000) should be kept
+      expect(units[0].id).toBe('1');
+      expect(units[0].price).toBe('1000');
+    });
+
+    it('should log a warning when duplicates are removed', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Duplicate units removed',
+        expect.objectContaining({
+          original: 3,
+          deduplicated: 2
+        })
+      );
+    });
+
+    it('should not log a warning when there are no duplicates', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      // The fixture has no duplicates, so no warn about duplicates
+      const warnCalls = mockLogger.warn.mock.calls;
+      const duplicateWarns = warnCalls.filter(
+        call => call[0] === 'Duplicate units removed'
+      );
+      expect(duplicateWarns).toHaveLength(0);
+    });
+
+    it('should filter out articles with no unit_code during deduplication', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // Article without unit_code should be filtered out
+      expect(units).toHaveLength(1);
+      expect(units[0].unit_code).toBe('VALID-1');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging behavior
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of parsed units', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 10 })
+      );
+    });
+
+    it('should log error when container is not found', () => {
+      const html = '<html><body><p>Nothing here</p></body></html>';
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.error).toHaveBeenCalledTimes(1);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Multiple units with varied data
+  // ---------------------------------------------------------------
+  describe('varied unit data', () => {
+    it('should handle unavailable units correctly', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="4001"
+            data-spaces-unit="UNAVAIL-1"
+            data-spaces-sort-price="2000"
+            data-spaces-available="false"
+            data-spaces-unavailable="true"
+            data-spaces-soonest=""
+            data-spaces-sort-date="">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.available).toBe('false');
+      expect(unit.unavailable).toBe('true');
+      expect(unit.soonest).toBe('');
+      expect(unit.date_available).toBe('');
+    });
+
+    it('should handle half bath counts', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // E3205 is the penthouse with 2.5 baths
+      const unit = units.find(u => u.unit_code === 'E3205');
+
+      expect(unit.bath_count).toBe('2.5');
+    });
+
+    it('should handle studio units (bed_count = 0)', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // W2603 is the studio with bed_count 0
+      const unit = units.find(u => u.unit_code === 'W2603');
+
+      expect(unit.bed_count).toBe('0');
+    });
+  });
+});

__tests__/scraper/recordScraperRun.test.js

@@ -0,0 +1,307 @@
+/**
+ * Tests for recordScraperRun() history persistence
+ *
+ * Covers:
+ * - insertOne is called with runData plus recordedAt timestamp
+ * - All runData fields are preserved (jobId, trigger, status, duration, etc.)
+ * - Error is caught and logged but NOT thrown (graceful failure)
+ * - Returns null on error instead of crashing
+ * - Returns insert result on success
+ * - recordedAt is a valid Date object
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require after implementation
+let recordScraperRun;
+
+let mongoServer;
+let client;
+let db;
+let logger;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ recordScraperRun } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+
+  // Create fresh mock logger for each test
+  logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+});
+
+describe('recordScraperRun', () => {
+  const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
+
+  // Sample runData matching the structure from runScrape()'s finally block
+  function createSampleRunData(overrides = {}) {
+    return {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      dryRun: false,
+      status: 'success',
+      startedAt: '2026-02-06T06:00:00.000Z',
+      completedAt: '2026-02-06T06:00:12.345Z',
+      duration: 12345,
+      unitsProcessed: 50,
+      pricesInserted: 48,
+      newUnitsCount: 3,
+      rentedUnitsCount: 1,
+      staleUnitsCount: 2,
+      errors: [],
+      ...overrides
+    };
+  }
+
+  // ---------------------------------------------------------------
+  // 1. insertOne is called with runData plus recordedAt timestamp
+  // ---------------------------------------------------------------
+  describe('insertOne with runData and recordedAt', () => {
+    it('should insert a document into the scraper_runs collection', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+
+    it('should include a recordedAt field in the inserted document', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. All runData fields are preserved
+  // ---------------------------------------------------------------
+  describe('preserving all runData fields', () => {
+    it('should preserve jobId, trigger, status, and duration', async () => {
+      const runData = createSampleRunData({
+        jobId: 'preserve-test-001',
+        trigger: 'scheduled',
+        status: 'failed',
+        duration: 9999
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.jobId).toBe('preserve-test-001');
+      expect(doc.trigger).toBe('scheduled');
+      expect(doc.status).toBe('failed');
+      expect(doc.duration).toBe(9999);
+    });
+
+    it('should preserve unit processing metrics', async () => {
+      const runData = createSampleRunData({
+        unitsProcessed: 42,
+        pricesInserted: 40,
+        newUnitsCount: 5,
+        rentedUnitsCount: 2,
+        staleUnitsCount: 3
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.unitsProcessed).toBe(42);
+      expect(doc.pricesInserted).toBe(40);
+      expect(doc.newUnitsCount).toBe(5);
+      expect(doc.rentedUnitsCount).toBe(2);
+      expect(doc.staleUnitsCount).toBe(3);
+    });
+
+    it('should preserve timing fields (startedAt, completedAt)', async () => {
+      const runData = createSampleRunData({
+        startedAt: '2026-02-06T10:00:00.000Z',
+        completedAt: '2026-02-06T10:00:15.500Z'
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
+      expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
+    });
+
+    it('should preserve dryRun flag', async () => {
+      const runData = createSampleRunData({ dryRun: true });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.dryRun).toBe(true);
+    });
+
+    it('should preserve errors array when it has entries', async () => {
+      const runData = createSampleRunData({
+        status: 'failed',
+        errors: ['Connection timeout', 'Retry exhausted']
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
+    });
+
+    it('should preserve empty errors array for successful runs', async () => {
+      const runData = createSampleRunData({ errors: [] });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Error is caught and logged but NOT thrown (graceful failure)
+  // ---------------------------------------------------------------
+  describe('graceful error handling', () => {
+    it('should NOT throw when insertOne fails', async () => {
+      const runData = createSampleRunData();
+
+      // Create a mock db that throws on insertOne
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      // This should NOT throw
+      await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
+    });
+
+    it('should log error message when insert fails', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await recordScraperRun(mockDb, runData, logger);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to record scraper run',
+        { errorMessage: 'Disk full' }
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Returns null on error instead of crashing
+  // ---------------------------------------------------------------
+  describe('return value on error', () => {
+    it('should return null when insertOne throws', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const result = await recordScraperRun(mockDb, runData, logger);
+      expect(result).toBeNull();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Returns insert result on success
+  // ---------------------------------------------------------------
+  describe('return value on success', () => {
+    it('should return the insertOne result object', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result).toBeDefined();
+      expect(result).not.toBeNull();
+    });
+
+    it('should return result with acknowledged property', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.acknowledged).toBe(true);
+    });
+
+    it('should return result with insertedId', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.insertedId).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. recordedAt is a valid Date object
+  // ---------------------------------------------------------------
+  describe('recordedAt field', () => {
+    it('should set recordedAt as a Date instance', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+
+    it('should set recordedAt close to current time', async () => {
+      const beforeTime = new Date();
+
+      const runData = createSampleRunData();
+      await recordScraperRun(db, runData, logger);
+
+      const afterTime = new Date();
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+      expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
+    });
+
+    it('should not overwrite any existing runData fields with recordedAt', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+
+      // Verify the original fields still exist alongside recordedAt
+      expect(doc.jobId).toBe(runData.jobId);
+      expect(doc.status).toBe(runData.status);
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+  });
+});

__tests__/scraper/scraperConfig.test.js

@@ -0,0 +1,192 @@
+/**
+ * Tests for config/scraper.js
+ * Tests scraper configuration constants and environment variable overrides
+ */
+
+describe('config/scraper', () => {
+  // Store original env values to restore after tests
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    // Clear all scraper-related env vars before each test
+    delete process.env.SCRAPER_SCHEDULE;
+    delete process.env.SCRAPER_TIMEZONE;
+    delete process.env.SCRAPER_ENABLED;
+    delete process.env.SCRAPER_TIMEOUT;
+    delete process.env.SCRAPER_UNITS_COLLECTION;
+    delete process.env.SCRAPER_PRICES_COLLECTION;
+    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
+    delete process.env.SCRAPER_RUNS_COLLECTION;
+
+    // Clear the require cache to reload config with fresh env
+    jest.resetModules();
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    process.env = originalEnv;
+  });
+
+  describe('TARGET_URL', () => {
+    test('should be set to the correct apartment listing URL', () => {
+      const config = require('../../config/scraper');
+      expect(config.TARGET_URL).toBe(
+        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
+      );
+    });
+  });
+
+  describe('SCRAPER_SCHEDULE', () => {
+    test('should default to "0 6 * * *" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
+    });
+  });
+
+  describe('SCRAPER_TIMEZONE', () => {
+    test('should default to "UTC" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+    });
+  });
+
+  describe('SCRAPER_ENABLED', () => {
+    test('should default to true when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be false when env var is set to "false"', () => {
+      process.env.SCRAPER_ENABLED = 'false';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(false);
+    });
+
+    test('should be true when env var is set to any other value', () => {
+      process.env.SCRAPER_ENABLED = 'true';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be true when env var is empty string', () => {
+      process.env.SCRAPER_ENABLED = '';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+  });
+
+  describe('SCRAPER_TIMEOUT', () => {
+    test('should default to 30000 when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(30000);
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEOUT = '60000';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(60000);
+    });
+
+    test('should parse string to integer', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
+      expect(config.SCRAPER_TIMEOUT).toBe(45000);
+    });
+  });
+
+  describe('USER_AGENT', () => {
+    test('should be a valid User-Agent string', () => {
+      const config = require('../../config/scraper');
+      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
+    });
+  });
+
+  describe('RETRY_CONFIG', () => {
+    test('should have maxRetries property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
+      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
+    });
+
+    test('should have baseDelay property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
+      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
+    });
+
+    test('should have timeout property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
+      expect(config.RETRY_CONFIG.timeout).toBe(30000);
+    });
+
+    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG.timeout).toBe(45000);
+    });
+  });
+
+  describe('COLLECTIONS', () => {
+    describe('default values', () => {
+      test('UNITS should default to "units_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
+      });
+
+      test('PRICES should default to "unit_prices_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
+      });
+
+      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
+      });
+
+      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
+      });
+    });
+
+    describe('env var overrides', () => {
+      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
+        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
+      });
+
+      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
+        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
+      });
+
+      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
+        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
+      });
+
+      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
+        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
+      });
+    });
+  });
+});

__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

__tests__/scraper/scraperService.test.js

@@ -0,0 +1,676 @@
+/**
+ * Tests for runScrape() orchestration function
+ *
+ * Covers:
+ * - Full workflow completes with mocked dependencies
+ * - Returns correct result structure with jobId, status, metrics
+ * - Handles dryRun option (skip DB writes)
+ * - Handles htmlContent option (use provided HTML)
+ * - Records scraper run to history on success
+ * - Records scraper run to history on failure
+ * - Catches and logs errors from fetchPage
+ * - Catches and logs errors from database operations
+ * - Calculates newUnitsCount and rentedUnitsCount correctly
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+let mongoServer;
+let client;
+let db;
+
+// We will require runScrape and recordScraperRun after implementation
+let runScrape;
+let recordScraperRun;
+
+// Store original module references so we can mock individual functions
+let scraperService;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  scraperService = require('../../services/scraperService');
+  ({ runScrape, recordScraperRun } = scraperService);
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all relevant collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+  // Reset all mocks
+  jest.restoreAllMocks();
+});
+
+// ============================================================
+// Helper: Create sample HTML with units
+// ============================================================
+function createSampleHtml(unitCodes) {
+  const articles = unitCodes.map(code => `
+    <article
+      data-spaces-id="100"
+      data-spaces-unit="${code}"
+      data-spaces-unit-id="200"
+      data-spaces-unit-floor="5"
+      data-spaces-sort-area="750"
+      data-spaces-sort-bed="1"
+      data-spaces-sort-bath="1"
+      data-spaces-sort-price="1500"
+      data-spaces-available="true"
+      data-spaces-unavailable="false"
+      data-spaces-soonest="Now"
+      data-spaces-sort-date="1700000000"
+      data-spaces-plan-id="10"
+      data-spaces-sort-plan-name="Studio"
+      data-spaces-obj="unit"
+      data-spaces-community="TestCommunity"
+      data-spaces-asset="1"
+      data-spaces-href="/unit/${code}"
+      data-spaces-inventory-href="/inventory/${code}"
+    >
+      <img src="http://example.com/${code}.jpg" />
+    </article>
+  `).join('\n');
+
+  return `
+    <html><body>
+      <section class="spaces__tab-unit">
+        ${articles}
+      </section>
+    </body></html>
+  `;
+}
+
+// ============================================================
+// Test: recordScraperRun
+// ============================================================
+describe('recordScraperRun', () => {
+  it('should insert a run record into scraper_runs collection', async () => {
+    const runData = {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      status: 'success',
+      startedAt: new Date().toISOString(),
+      completedAt: new Date().toISOString(),
+      duration: 1234,
+      unitsProcessed: 10,
+      pricesInserted: 8,
+      errors: []
+    };
+
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(db, runData, mockLogger);
+
+    expect(result).toBeTruthy();
+    expect(result.insertedId).toBeTruthy();
+
+    // Verify it was inserted
+    const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
+    expect(saved).toBeTruthy();
+    expect(saved.status).toBe('success');
+    expect(saved.recordedAt).toBeInstanceOf(Date);
+  });
+
+  it('should not throw when insert fails', async () => {
+    // Pass null db to cause an error
+    const runData = { jobId: 'test-fail', status: 'success' };
+
+    // Should not throw
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(null, runData, mockLogger);
+    expect(result).toBeNull();
+    expect(mockLogger.error).toHaveBeenCalled();
+  });
+});
+
+// ============================================================
+// Test: runScrape - Full workflow
+// ============================================================
+describe('runScrape', () => {
+  it('should complete full workflow with mocked dependencies', async () => {
+    const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
+
+    const result = await runScrape(db, {
+      trigger: 'manual',
+      jobId: 'test-full-workflow',
+      htmlContent: html
+    });
+
+    expect(result).toBeTruthy();
+    expect(result.jobId).toBe('test-full-workflow');
+    expect(result.status).toBe('success');
+    expect(result.trigger).toBe('manual');
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.errors).toEqual([]);
+  });
+
+  it('should return correct result structure with jobId, status, metrics', async () => {
+    const html = createSampleHtml(['UNIT-X']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-structure',
+      htmlContent: html
+    });
+
+    // Verify all expected fields exist
+    expect(result).toHaveProperty('jobId', 'test-structure');
+    expect(result).toHaveProperty('trigger', 'manual');
+    expect(result).toHaveProperty('dryRun', false);
+    expect(result).toHaveProperty('status', 'success');
+    expect(result).toHaveProperty('startedAt');
+    expect(result).toHaveProperty('completedAt');
+    expect(result).toHaveProperty('duration');
+    expect(result).toHaveProperty('unitsProcessed');
+    expect(result).toHaveProperty('pricesInserted');
+    expect(result).toHaveProperty('newUnitsCount');
+    expect(result).toHaveProperty('rentedUnitsCount');
+    expect(result).toHaveProperty('staleUnitsCount');
+    expect(result).toHaveProperty('errors');
+
+    // Verify types
+    expect(typeof result.duration).toBe('number');
+    expect(result.duration).toBeGreaterThanOrEqual(0);
+    expect(typeof result.startedAt).toBe('string');
+    expect(typeof result.completedAt).toBe('string');
+    expect(Array.isArray(result.errors)).toBe(true);
+  });
+
+  it('should generate a jobId when not provided', async () => {
+    const html = createSampleHtml(['UNIT-GEN']);
+
+    const result = await runScrape(db, {
+      htmlContent: html
+    });
+
+    expect(result.jobId).toBeTruthy();
+    expect(typeof result.jobId).toBe('string');
+    // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+    expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
+  });
+
+  // ============================================================
+  // Test: dryRun option
+  // ============================================================
+  it('should skip DB writes when dryRun is true', async () => {
+    const html = createSampleHtml(['DRY-A', 'DRY-B']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-dry-run',
+      htmlContent: html,
+      dryRun: true
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.dryRun).toBe(true);
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.pricesInserted).toBe(0);
+    expect(result.staleUnitsCount).toBe(0);
+
+    // Verify no units were written to the units collection
+    const unitsCount = await db.collection('units_migration_test').countDocuments();
+    expect(unitsCount).toBe(0);
+
+    // Verify no prices were written
+    const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
+    expect(pricesCount).toBe(0);
+  });
+
+  // ============================================================
+  // Test: htmlContent option
+  // ============================================================
+  it('should use provided HTML instead of fetching when htmlContent is given', async () => {
+    const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
+
+    // Mock axios.get to track if HTTP fetch is attempted
+    const axios = require('axios');
+    const axiosSpy = jest.spyOn(axios, 'get');
+
+    const result = await runScrape(db, {
+      jobId: 'test-html-content',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(3);
+    // axios.get should NOT have been called since we provided htmlContent
+    expect(axiosSpy).not.toHaveBeenCalled();
+  });
+
+  // ============================================================
+  // Test: Records scraper run on success
+  // ============================================================
+  it('should record scraper run to history on success', async () => {
+    const html = createSampleHtml(['REC-A']);
+
+    await runScrape(db, {
+      jobId: 'test-record-success',
+      htmlContent: html
+    });
+
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('success');
+    expect(runRecord.recordedAt).toBeInstanceOf(Date);
+    expect(runRecord.unitsProcessed).toBe(1);
+  });
+
+  // ============================================================
+  // Test: Records scraper run on failure
+  // ============================================================
+  it('should record scraper run to history on failure', async () => {
+    const html = createSampleHtml(['FAIL-A']);
+
+    // Create a db proxy that throws on bulkWrite (used by upsertUnits)
+    // but allows other operations (like scraper_runs insertOne) to pass through
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'units_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Database connection lost');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-record-failure',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Database connection lost');
+
+    // The run should still be recorded (via the real db passed through for scraper_runs)
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+    expect(runRecord.errors).toContain('Database connection lost');
+  });
+
+  // ============================================================
+  // Test: Catches errors from fetchPage
+  // ============================================================
+  it('should catch and handle errors from fetchPage', async () => {
+    // Mock axios.get to throw a network error (fetchPage uses axios internally)
+    const axios = require('axios');
+    jest.spyOn(axios, 'get').mockRejectedValue(
+      new Error('Network timeout')
+    );
+
+    const result = await runScrape(db, {
+      jobId: 'test-fetch-error'
+      // No htmlContent, so it will call fetchPage which uses axios
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Network timeout');
+    expect(result.unitsProcessed).toBe(0);
+  });
+
+  // ============================================================
+  // Test: Catches errors from database operations
+  // ============================================================
+  it('should catch and handle errors from database operations', async () => {
+    const html = createSampleHtml(['DB-ERR']);
+
+    // Create a db proxy that throws on the prices collection bulkWrite
+    // Use Proxy to properly delegate all methods to the real collection
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'unit_prices_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Write concern timeout');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-db-error',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Write concern timeout');
+
+    // Run should still be recorded despite error
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+  });
+
+  // ============================================================
+  // Test: Calculates newUnitsCount and rentedUnitsCount correctly
+  // ============================================================
+  it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
+    // First, simulate yesterday's data by inserting price records for yesterday
+    const today = new Date();
+    const yesterday = new Date(today);
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const yesterdayStr = yesterday.toISOString().split('T')[0];
+
+    // Yesterday had units: OLD-A, OLD-B, OLD-C
+    await db.collection('unit_prices_migration_test').insertMany([
+      { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
+      { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
+      { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
+    ]);
+
+    // Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
+    const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-calc-changes',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    // NEW-D is new (not in yesterday's data)
+    expect(result.newUnitsCount).toBe(1);
+    // OLD-C was in yesterday's data but not in today's scrape
+    expect(result.rentedUnitsCount).toBe(1);
+    expect(result.unitsProcessed).toBe(3);
+  });
+
+  // ============================================================
+  // Test: Default trigger is 'manual'
+  // ============================================================
+  it('should default trigger to manual', async () => {
+    const html = createSampleHtml(['DEF-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-default-trigger',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('manual');
+  });
+
+  // ============================================================
+  // Test: Supports scheduled trigger
+  // ============================================================
+  it('should support scheduled trigger', async () => {
+    const html = createSampleHtml(['SCHED-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-scheduled-trigger',
+      trigger: 'scheduled',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('scheduled');
+  });
+
+  // ============================================================
+  // Test: Handles empty HTML (no units found)
+  // ============================================================
+  it('should handle HTML with no units gracefully', async () => {
+    const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+
+    const result = await runScrape(db, {
+      jobId: 'test-empty-html',
+      htmlContent: emptyHtml
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(0);
+    expect(result.errors).toContain('No units found in HTML');
+  });
+});
+
+// ============================================================
+// Test: sanitizeError - Error message sanitization
+// ============================================================
+describe('sanitizeError', () => {
+  let sanitizeError;
+
+  beforeAll(() => {
+    ({ sanitizeError } = require('../../services/scraperService'));
+  });
+
+  // ----------------------------------------------------------
+  // File path sanitization
+  // ----------------------------------------------------------
+  describe('file path sanitization', () => {
+    it('should remove Unix absolute file paths from error messages', () => {
+      const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/home\/user/);
+      expect(sanitized.message).toContain('ENOENT');
+    });
+
+    it('should remove /var paths from error messages', () => {
+      const error = new Error('Failed to read /var/app/current/data/secrets.yml');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/var\/app/);
+      expect(sanitized.message).toContain('Failed to read');
+    });
+
+    it('should remove Windows-style file paths from error messages', () => {
+      const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/C:\\Users/);
+      expect(sanitized.message).toContain('Cannot find module');
+    });
+
+    it('should remove /tmp and /usr paths from error messages', () => {
+      const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/tmp\//);
+      expect(sanitized.message).not.toMatch(/\/usr\//);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // MongoDB connection string sanitization
+  // ----------------------------------------------------------
+  describe('connection string sanitization', () => {
+    it('should redact mongodb:// connection strings', () => {
+      const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('s3cretP4ss');
+      expect(sanitized.message).not.toContain('admin:');
+      expect(sanitized.message).toContain('Connection failed');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact mongodb+srv:// connection strings', () => {
+      const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('password123');
+      expect(sanitized.message).not.toContain('user:');
+      expect(sanitized.message).toContain('Timeout connecting to');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact connection string without credentials', () => {
+      const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Credential / secret sanitization
+  // ----------------------------------------------------------
+  describe('credential sanitization', () => {
+    it('should redact common environment variable patterns', () => {
+      const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('sk-abc123xyz');
+      expect(sanitized.message).not.toContain('bearer-9876');
+    });
+
+    it('should redact password patterns', () => {
+      const error = new Error('Auth failed with password=MyS3cret!');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('MyS3cret!');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Error type preservation
+  // ----------------------------------------------------------
+  describe('error type preservation', () => {
+    it('should preserve the error type/name', () => {
+      const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TypeError');
+    });
+
+    it('should preserve custom error names', () => {
+      const error = new Error('Timeout at /var/app/scraper.js:100');
+      error.name = 'TimeoutError';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TimeoutError');
+    });
+
+    it('should preserve error name for RangeError', () => {
+      const error = new RangeError('Maximum call stack size exceeded');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('RangeError');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // General description preserved for debugging
+  // ----------------------------------------------------------
+  describe('general description preservation', () => {
+    it('should preserve a useful general description', () => {
+      const error = new Error('Network timeout after 30000ms');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('Network timeout after 30000ms');
+    });
+
+    it('should preserve error description when no sensitive data present', () => {
+      const error = new Error('Request failed with status code 500');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toBe('Request failed with status code 500');
+    });
+
+    it('should return a useful message even after heavy sanitization', () => {
+      const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('ECONNREFUSED');
+      expect(sanitized.message.length).toBeGreaterThan(5);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Stack trace sanitization
+  // ----------------------------------------------------------
+  describe('stack trace removal', () => {
+    it('should remove file paths from stack traces', () => {
+      const error = new Error('Something failed');
+      error.stack = 'Error: Something failed\n    at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n    at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).not.toMatch(/\/home\/user/);
+      expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
+    });
+
+    it('should handle errors without stack trace', () => {
+      const error = new Error('No stack');
+      error.stack = undefined;
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).toBeUndefined();
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Integration: sanitization applied before recordScraperRun()
+  // ----------------------------------------------------------
+  describe('sanitization before recordScraperRun()', () => {
+    it('should store sanitized error message in scraper_runs on failure', async () => {
+      const html = createSampleHtml(['SANITIZE-A']);
+
+      // Create a db proxy that throws an error containing sensitive info
+      const sensitiveError = new Error(
+        'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
+      );
+      sensitiveError.name = 'MongoServerError';
+
+      const faultyDb = {
+        collection: (name) => {
+          const realCollection = db.collection(name);
+          if (name === 'units_migration_test') {
+            return new Proxy(realCollection, {
+              get(target, prop) {
+                if (prop === 'bulkWrite') {
+                  return async () => { throw sensitiveError; };
+                }
+                const value = target[prop];
+                if (typeof value === 'function') {
+                  return value.bind(target);
+                }
+                return value;
+              }
+            });
+          }
+          return realCollection;
+        }
+      };
+
+      const result = await runScrape(faultyDb, {
+        jobId: 'test-sanitized-error',
+        htmlContent: html
+      });
+
+      expect(result.status).toBe('failed');
+
+      // Verify the error stored in result.errors is sanitized
+      const errorMsg = result.errors[0];
+      expect(errorMsg).not.toContain('SuperSecret');
+      expect(errorMsg).not.toContain('admin:');
+      expect(errorMsg).not.toContain('/home/deploy/');
+      expect(errorMsg).toContain('MongoServerError');
+
+      // Verify the error stored in scraper_runs is sanitized
+      const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
+      expect(runRecord).toBeTruthy();
+      expect(runRecord.status).toBe('failed');
+      const storedError = runRecord.errors[0];
+      expect(storedError).not.toContain('SuperSecret');
+      expect(storedError).not.toContain('admin:');
+      expect(storedError).not.toContain('/home/deploy/');
+    });
+  });
+});

__tests__/scraper/updateDailySummary.test.js

@@ -0,0 +1,561 @@
+/**
+ * Tests for updateDailySummary() daily summary aggregation
+ *
+ * Covers:
+ * - Upsert by date key (YYYY-MM-DD)
+ * - Summary includes new_units, rented_units, stale_units arrays
+ * - Count fields are calculated correctly
+ * - net_change = newUnits.length - rentedUnits.length
+ * - total_available_today is set correctly
+ * - total_available_yesterday fetched from previous day
+ * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+ * - turnover_rate is 0 when yesterdayTotal is 0
+ * - Error handling when updateOne fails
+ * - Logging of summary update details
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require updateDailySummary after implementation
+let updateDailySummary;
+
+let mongoServer;
+let client;
+let db;
+
+const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ updateDailySummary } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('updateDailySummary', () => {
+  // ---------------------------------------------------------------
+  // 1. Upsert by date key
+  // ---------------------------------------------------------------
+  describe('upsert by date key', () => {
+    it('should insert a new summary when no document exists for the date', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 2,
+        totalAvailable: 15
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have upserted (created new)
+      expect(result.upsertedCount).toBe(1);
+
+      // Verify the document in the database
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc).not.toBeNull();
+      expect(doc.date).toBe('2026-02-05');
+    });
+
+    it('should update an existing summary for the same date (idempotent upsert)', async () => {
+      const logger = createMockLogger();
+
+      // Insert initial summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-05',
+        new_units: ['A101'],
+        rented_units: [],
+        total_available_today: 10
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 12
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have matched and modified (not upserted)
+      expect(result.matchedCount).toBe(1);
+
+      // Verify updated values
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
+      expect(doc.total_available_today).toBe(12);
+
+      // Verify only one document exists for this date
+      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Summary includes new_units, rented_units, stale_units arrays
+  // ---------------------------------------------------------------
+  describe('summary arrays', () => {
+    it('should store new_units, rented_units, and stale_units arrays', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202', 'B203'],
+        staleUnitsCount: 3,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102']);
+      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
+      expect(doc.stale_units).toEqual([]);
+    });
+
+    it('should handle empty arrays for new_units and rented_units', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 25
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual([]);
+      expect(doc.rented_units).toEqual([]);
+      expect(doc.stale_units).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Count fields are calculated correctly
+  // ---------------------------------------------------------------
+  describe('count fields', () => {
+    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 5,
+        totalAvailable: 30
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(3);
+      expect(doc.rented_units_count).toBe(1);
+      expect(doc.stale_units_count).toBe(5);
+    });
+
+    it('should set all count fields to 0 when arrays are empty', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(0);
+      expect(doc.rented_units_count).toBe(0);
+      expect(doc.stale_units_count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. net_change = newUnits.length - rentedUnits.length
+  // ---------------------------------------------------------------
+  describe('net_change calculation', () => {
+    it('should calculate positive net_change when more units added than rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(2); // 3 - 1 = 2
+    });
+
+    it('should calculate negative net_change when more units rented than added', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
+    });
+
+    it('should calculate zero net_change when equal new and rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. total_available_today is set correctly
+  // ---------------------------------------------------------------
+  describe('total_available_today', () => {
+    it('should store the totalAvailable value as total_available_today', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 42
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(42);
+    });
+
+    it('should handle totalAvailable of 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 0
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. total_available_yesterday fetched from previous day
+  // ---------------------------------------------------------------
+  describe('total_available_yesterday', () => {
+    it('should fetch total_available_today from previous day summary', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 35,
+        new_units: [],
+        rented_units: []
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 35
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(35);
+    });
+
+    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(0);
+    });
+
+    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
+      const logger = createMockLogger();
+
+      // Insert March 31 summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-03-31',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-04-01',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 48
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
+      expect(doc.total_available_yesterday).toBe(50);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+  // ---------------------------------------------------------------
+  describe('turnover_rate calculation', () => {
+    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 50 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
+        staleUnitsCount: 0,
+        totalAvailable: 46
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 5 rented / 50 yesterday = 10%
+      expect(doc.turnover_rate).toBe(10);
+    });
+
+    it('should round turnover_rate to 2 decimal places', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 30 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 30
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 28
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
+      expect(doc.turnover_rate).toBe(6.67);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. turnover_rate is 0 when yesterdayTotal is 0
+  // ---------------------------------------------------------------
+  describe('turnover_rate edge cases', () => {
+    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 0 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 0
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Error handling when updateOne fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw and log error when updateOne fails', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      // Create a mock db that throws on updateOne
+      const mockCollection = {
+        findOne: jest.fn().mockResolvedValue(null),
+        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(
+        updateDailySummary(mockDb, summaryData, logger)
+      ).rejects.toThrow('Write concern timeout');
+
+      // Verify error was logged
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to update daily summary',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Logging of summary update details
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log summary update details on success', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 18
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Daily summary updated',
+        expect.objectContaining({
+          date: '2026-02-05',
+          newUnits: 2,
+          rentedUnits: 1,
+          totalAvailable: 18
+        })
+      );
+    });
+
+    it('should include timestamp field in the stored summary', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.timestamp).toBeDefined();
+      expect(typeof doc.timestamp).toBe('string');
+      // Verify it is a valid ISO timestamp
+      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
+    });
+  });
+});

__tests__/scraper/upsertUnits.test.js

@@ -0,0 +1,498 @@
+/**
+ * Tests for upsertUnits() bulk operation
+ *
+ * Covers:
+ * - bulkWrite is called with correct updateOne operations
+ * - upsert: true is set for all operations
+ * - $set updates last_scraped and data_source fields
+ * - $setOnInsert sets first_seen for new units
+ * - Empty units array logs warning and returns early
+ * - ordered: false is set for parallel execution
+ * - Error handling when bulkWrite fails
+ * - Logging of matched, modified, and upserted counts
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// We will require upsertUnits after implementation
+let upsertUnits;
+
+let mongoServer;
+let client;
+let db;
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ upsertUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the units collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('upsertUnits', () => {
+  const UNITS_COLLECTION = 'units_migration_test';
+
+  // ---------------------------------------------------------------
+  // 1. bulkWrite is called with correct updateOne operations
+  // ---------------------------------------------------------------
+  describe('bulkWrite operations', () => {
+    it('should create updateOne operations for each unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
+        { unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.upsertedCount).toBe(2);
+
+      // Verify documents exist in the collection
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'B202']);
+    });
+
+    it('should filter by unit_code as the unique key', async () => {
+      const logger = createMockLogger();
+
+      // Insert a unit first
+      const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
+      await upsertUnits(db, units, logger);
+
+      // Upsert again with updated price
+      const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
+      const result = await upsertUnits(db, updatedUnits, logger);
+
+      // Should match and modify, not insert new
+      expect(result.matchedCount).toBe(1);
+      expect(result.upsertedCount).toBe(0);
+
+      // Verify only one document exists
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+      expect(docs[0].price).toBe(1400);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. upsert: true is set for all operations
+  // ---------------------------------------------------------------
+  describe('upsert behavior', () => {
+    it('should insert new units when they do not exist (upsert: true)', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'NEW-001', price: 1100 },
+        { unit_code: 'NEW-002', price: 1200 },
+        { unit_code: 'NEW-003', price: 1300 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(3);
+
+      // Verify all three are in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+
+    it('should update existing units without creating duplicates', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
+
+      // Second upsert with same unit_code
+      await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. $set updates last_scraped and data_source fields
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set last_scraped timestamp on every upsert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'T100', price: 1000 }];
+      await upsertUnits(db, units, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
+      expect(doc.last_scraped).toBeDefined();
+      expect(typeof doc.last_scraped).toBe('string');
+      // The last_scraped should be between before and after timestamps
+      expect(doc.last_scraped >= beforeTime).toBe(true);
+      expect(doc.last_scraped <= afterTime).toBe(true);
+    });
+
+    it('should set data_source to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'T200', price: 1100 }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should update last_scraped on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const firstScraped = doc1.last_scraped;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const secondScraped = doc2.last_scraped;
+
+      expect(secondScraped > firstScraped).toBe(true);
+    });
+
+    it('should spread all unit fields into the $set operation', async () => {
+      const logger = createMockLogger();
+      const units = [{
+        unit_code: 'T400',
+        price: 1500,
+        bed_count: 2,
+        bath_count: 1.5,
+        area: 900,
+        floor: 3,
+        plan_name: 'Luxury',
+        community: 'Tower A'
+      }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
+      expect(doc.price).toBe(1500);
+      expect(doc.bed_count).toBe(2);
+      expect(doc.bath_count).toBe(1.5);
+      expect(doc.area).toBe(900);
+      expect(doc.floor).toBe(3);
+      expect(doc.plan_name).toBe('Luxury');
+      expect(doc.community).toBe('Tower A');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. $setOnInsert sets first_seen for new units
+  // ---------------------------------------------------------------
+  describe('$setOnInsert for first_seen', () => {
+    it('should set first_seen on initial insert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
+      expect(doc.first_seen).toBeDefined();
+      expect(typeof doc.first_seen).toBe('string');
+      expect(doc.first_seen >= beforeTime).toBe(true);
+    });
+
+    it('should NOT update first_seen on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+      const originalFirstSeen = doc1.first_seen;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert (update)
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+
+      // first_seen should remain unchanged
+      expect(doc2.first_seen).toBe(originalFirstSeen);
+    });
+
+    it('should set different first_seen for different units inserted at different times', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
+
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
+
+      // Both should have first_seen, but different values
+      expect(doc1.first_seen).toBeDefined();
+      expect(doc2.first_seen).toBeDefined();
+      expect(doc2.first_seen > doc1.first_seen).toBe(true);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty units array logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty units array', () => {
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
+    });
+
+    it('should return early with zero counts for empty array', async () => {
+      const logger = createMockLogger();
+
+      const result = await upsertUnits(db, [], logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(result.upsertedCount).toBe(0);
+    });
+
+    it('should not perform any database writes for empty array', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. ordered: false is set for parallel execution
+  // ---------------------------------------------------------------
+  describe('ordered: false for parallel execution', () => {
+    it('should successfully process all units even if one has a conflict', async () => {
+      const logger = createMockLogger();
+
+      // Insert multiple units - ordered: false means all operations execute
+      // even if some fail. We verify by checking all valid units are present.
+      const units = [
+        { unit_code: 'P100', price: 1000 },
+        { unit_code: 'P200', price: 1100 },
+        { unit_code: 'P300', price: 1200 },
+        { unit_code: 'P400', price: 1300 },
+        { unit_code: 'P500', price: 1400 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(5);
+
+      // All 5 units should be in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+
+    it('should handle a mix of new inserts and existing updates', async () => {
+      const logger = createMockLogger();
+
+      // First, insert some units
+      await upsertUnits(db, [
+        { unit_code: 'M100', price: 900 },
+        { unit_code: 'M200', price: 1000 }
+      ], logger);
+
+      // Now upsert a mix of existing and new units
+      const mixedUnits = [
+        { unit_code: 'M100', price: 950 },  // existing - update
+        { unit_code: 'M200', price: 1050 }, // existing - update
+        { unit_code: 'M300', price: 1100 }, // new - insert
+      ];
+
+      const result = await upsertUnits(db, mixedUnits, logger);
+
+      expect(result.matchedCount).toBe(2);
+      expect(result.upsertedCount).toBe(1);
+
+      // Total should be 3 documents
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E100', price: 1000 }];
+
+      // Create a mock db that throws on bulkWrite
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E200', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await upsertUnits(mockDb, units, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to upsert units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E300', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Logging of matched, modified, and upserted counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log matched, modified, and upserted counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L100', price: 1000 },
+        { unit_code: 'L200', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: expect.any(Number),
+          modified: expect.any(Number),
+          upserted: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L300', price: 1000 },
+        { unit_code: 'L400', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      // For new inserts: matched=0, modified=0, upserted=2
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 0,
+          modified: 0,
+          upserted: 2
+        })
+      );
+    });
+
+    it('should log correct counts for updates to existing units', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
+
+      // Reset mock to capture only the second call
+      logger.info.mockClear();
+
+      // Update existing
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 1,
+          modified: 1,
+          upserted: 0
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the bulkWrite result object', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'R100', price: 1000 }];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // bulkWrite result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('upsertedCount');
+    });
+  });
+});

__tests__/setup.js

@@ -0,0 +1,13 @@
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+module.exports = async () => {
+  // Create an in-memory MongoDB instance for testing
+  const mongod = await MongoMemoryServer.create();
+  const uri = mongod.getUri();
+
+  // Store the URI and instance globally so tests can access them
+  global.__MONGOD__ = mongod;
+  process.env.MONGO_URI = uri;
+  process.env.JWT_SECRET = 'test-jwt-secret-for-testing-only';
+  process.env.NODE_ENV = 'test';
+};

__tests__/setupAfterEnv.js

@@ -0,0 +1,11 @@
+// Increase timeout for tests that interact with MongoDB
+jest.setTimeout(30000);
+
+// Suppress console logs during tests (optional - comment out for debugging)
+// global.console = {
+//   ...console,
+//   log: jest.fn(),
+//   debug: jest.fn(),
+//   info: jest.fn(),
+//   warn: jest.fn(),
+// };

__tests__/teardown.js

@@ -0,0 +1,6 @@
+module.exports = async () => {
+  // Stop the in-memory MongoDB instance
+  if (global.__MONGOD__) {
+    await global.__MONGOD__.stop();
+  }
+};

config/auth.js

@@ -0,0 +1,24 @@
+// Google OAuth and JWT configuration
+module.exports = {
+  google: {
+    clientID: process.env.GOOGLE_CLIENT_ID,
+    clientSecret: process.env.GOOGLE_CLIENT_SECRET,
+    callbackURL: process.env.GOOGLE_CALLBACK_URL,
+    scope: ['profile', 'email']
+  },
+  jwt: {
+    secret: process.env.JWT_SECRET,
+    expiresIn: '7d',
+    refreshThreshold: 24 * 60 * 60  // Refresh if token is older than 1 day (in seconds)
+  },
+  cookie: {
+    name: 'auth_token',
+    options: {
+      httpOnly: true,
+      secure: process.env.NODE_ENV === 'production',
+      sameSite: 'lax',
+      maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
+      domain: process.env.NODE_ENV === 'production' ? '.maverickapplications.com' : undefined
+    }
+  }
+};

config/scraper.js

@@ -0,0 +1,40 @@
+/**
+ * Scraper Configuration Module
+ *
+ * Contains all constants and settings for the apartment price scraper.
+ * All values can be overridden via environment variables for deployment flexibility.
+ *
+ * See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
+ */
+
+module.exports = {
+  // Target URL for apartment listings
+  TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
+
+  // Scheduling configuration
+  SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
+  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
+  SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
+
+  // HTTP settings
+  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
+
+  // Retry configuration for HTTP requests
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
+    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
+  },
+
+  // Graceful shutdown timeout (how long to wait for running job before force-stopping)
+  SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
+
+  // MongoDB collection names (environment variable overrides for development isolation)
+  COLLECTIONS: {
+    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
+    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
+    DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
+    SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
+  }
+};

cosign.pub

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEBOFl4SHzoZVM59+668t2tecTyKM+
+cE4zaOkbFO30u7d2+bjpwPnbYrTutKmAeehYzCAVW/OYea7ML0cW2YCSCQ==
+-----END PUBLIC KEY-----

docker-compose.yml

@@ -1,12 +1,21 @@
 services:
   apartment-api:
+    image: ${IMAGE:-apartment-api:latest}
     build: .
     container_name: apartment-api
     restart: unless-stopped
+    env_file:
+      - .env
     environment:
       - NODE_ENV=production
-      - PORT=8080  # Changed from 3000 to 8080
+      - PORT=8080
-      - MONGO_URI=mongodb://admin:password123@mongodb:27017
+      - MONGO_URI=${MONGO_URI}
+      - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID}
+      - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET}
+      - GOOGLE_CALLBACK_URL=${GOOGLE_CALLBACK_URL}
+      - JWT_SECRET=${JWT_SECRET}
+      - FRONTEND_URL=${FRONTEND_URL}
+      - ACTIVITY_LOG_LEVEL=${ACTIVITY_LOG_LEVEL:-all}
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=traefik"

eslint.config.mjs

@@ -0,0 +1,34 @@
+import js from '@eslint/js';
+import globals from 'globals';
+
+export default [
+  js.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'commonjs',
+      globals: {
+        ...globals.node,
+        ...globals.jest,
+      },
+    },
+    rules: {
+      // Allow unused vars with underscore prefix, and unused function args
+      'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
+      'no-console': 'off',
+      // Formatting rules as warnings (won't fail CI, but will be reported)
+      'semi': ['warn', 'always'],
+      'no-multiple-empty-lines': ['warn', { max: 2 }],
+      'eol-last': ['warn', 'always'],
+      'no-trailing-spaces': 'warn',
+    },
+  },
+  {
+    ignores: [
+      'node_modules/**',
+      'coverage/**',
+      '*.min.js',
+      'eslint.config.mjs',
+    ],
+  },
+];

jest.config.js

@@ -0,0 +1,17 @@
+module.exports = {
+  testEnvironment: 'node',
+  testMatch: ['**/__tests__/**/*.test.js'],
+  collectCoverageFrom: [
+    '**/*.js',
+    '!**/node_modules/**',
+    '!**/coverage/**',
+    '!jest.config.js'
+  ],
+  coverageDirectory: 'coverage',
+  verbose: true,
+  testTimeout: 30000,
+  // Setup file to handle MongoDB memory server lifecycle
+  globalSetup: './__tests__/setup.js',
+  globalTeardown: './__tests__/teardown.js',
+  setupFilesAfterEnv: ['./__tests__/setupAfterEnv.js']
+};

jobs/scraperJob.js

@@ -0,0 +1,275 @@
+const cron = require('node-cron');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { runScrape } = require('../services/scraperService');
+const { createLogger } = require('../services/scraperLogger');
+
+// In-process mutex state
+let isRunning = false;
+let currentJobId = null;
+
+// Scheduler state
+let scheduledJob = null;
+
+// Shutdown state
+let shuttingDown = false;
+let runningJobPromise = null;
+
+/**
+ * Check if scraper is currently running
+ * @returns {boolean}
+ */
+function isScraperRunning() {
+  return isRunning;
+}
+
+/**
+ * Get current job ID if running
+ * @returns {string|null}
+ */
+function getCurrentJobId() {
+  return currentJobId;
+}
+
+/**
+ * Acquire the scraper lock
+ * @param {string} jobId - Job ID to set
+ * @returns {boolean} True if lock acquired
+ */
+function acquireLock(jobId) {
+  if (isRunning || shuttingDown) {
+    return false;
+  }
+  isRunning = true;
+  currentJobId = jobId;
+  return true;
+}
+
+/**
+ * Release the scraper lock
+ */
+function releaseLock() {
+  isRunning = false;
+  currentJobId = null;
+}
+
+/**
+ * Get the configured schedule expression
+ * @returns {string} Cron expression or 'disabled'
+ */
+function getScheduleExpression() {
+  if (!config.SCRAPER_ENABLED) {
+    return 'disabled';
+  }
+  return config.SCRAPER_SCHEDULE;
+}
+
+/**
+ * Calculate next scheduled run time
+ * @returns {string|null} ISO timestamp or null if disabled
+ */
+function getNextScheduledRun() {
+  if (!config.SCRAPER_ENABLED || !scheduledJob) {
+    return null;
+  }
+
+  const { CronExpressionParser } = require('cron-parser');
+  try {
+    const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
+      tz: config.SCRAPER_TIMEZONE
+    });
+    return interval.next().toISOString();
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Check if schedule runs more frequently than 1 hour
+ * @param {string} schedule - Cron expression
+ * @returns {boolean} True if schedule is too frequent
+ */
+function isScheduleTooFrequent(schedule) {
+  const parts = schedule.trim().split(/\s+/);
+
+  if (parts.length < 5) return false;
+
+  const minuteField = parts[0];
+
+  // If minute field is */N with N < 60, it runs more than once per hour
+  if (/^\*\/\d+$/.test(minuteField)) {
+    const interval = parseInt(minuteField.substring(2), 10);
+    if (interval < 60) return true;
+  }
+
+  // If minute field is *, it runs every minute
+  if (minuteField === '*') return true;
+
+  return false;
+}
+
+/**
+ * Initialize the scraper scheduler
+ * @param {Db} db - MongoDB database instance
+ */
+function initializeScheduler(db) {
+  const logger = createLogger('scheduler');
+
+  // Check if scheduling is enabled
+  if (!config.SCRAPER_ENABLED) {
+    logger.info('Scraper scheduling is disabled');
+    return;
+  }
+
+  // Validate cron expression
+  if (!cron.validate(config.SCRAPER_SCHEDULE)) {
+    logger.error('Invalid cron schedule expression', {
+      schedule: config.SCRAPER_SCHEDULE
+    });
+    logger.warn('Falling back to default schedule: 0 6 * * *');
+    config.SCRAPER_SCHEDULE = '0 6 * * *';
+  }
+
+  // Validate minimum interval (1 hour)
+  if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
+    logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
+      originalSchedule: config.SCRAPER_SCHEDULE
+    });
+    config.SCRAPER_SCHEDULE = '0 * * * *';
+  }
+
+  // Create the scheduled job
+  scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
+    const jobId = crypto.randomUUID();
+    const jobLogger = createLogger(jobId);
+
+    jobLogger.info('Scheduled scrape triggered');
+
+    // Check if already running or shutting down
+    if (!acquireLock(jobId)) {
+      jobLogger.warn('Skipped - scrape already in progress');
+      return;
+    }
+
+    try {
+      const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
+      runningJobPromise = jobExecution;
+      await jobExecution;
+    } catch (error) {
+      jobLogger.error('Scheduled scrape failed', {
+        errorType: error.name,
+        errorMessage: error.message
+      });
+    } finally {
+      runningJobPromise = null;
+      releaseLock();
+    }
+  }, {
+    timezone: config.SCRAPER_TIMEZONE,
+    scheduled: true
+  });
+
+  logger.info('Scraper scheduler initialized', {
+    schedule: config.SCRAPER_SCHEDULE,
+    timezone: config.SCRAPER_TIMEZONE,
+    nextRun: getNextScheduledRun()
+  });
+}
+
+/**
+ * Stop the scheduler (for graceful shutdown)
+ */
+function stopScheduler() {
+  if (scheduledJob) {
+    scheduledJob.stop();
+    scheduledJob = null;
+  }
+}
+
+/**
+ * Check if the scraper is in the process of shutting down
+ * @returns {boolean}
+ */
+function isShuttingDown() {
+  return shuttingDown;
+}
+
+/**
+ * Perform a graceful shutdown of the scraper
+ * - Stops the cron scheduler to prevent new jobs
+ * - Waits for any running job to complete (with timeout)
+ * - Releases the mutex lock
+ * - Logs shutdown progress
+ * @returns {Promise<void>}
+ */
+async function gracefulShutdown() {
+  const logger = createLogger('shutdown');
+
+  logger.info('Shutdown initiated');
+
+  // Mark as shutting down to prevent new jobs
+  shuttingDown = true;
+
+  // Stop the cron scheduler
+  stopScheduler();
+
+  // Wait for running job to complete (with timeout)
+  if (isRunning && runningJobPromise) {
+    logger.info('Waiting for running job to complete', {
+      jobId: currentJobId,
+      timeout: config.SHUTDOWN_TIMEOUT
+    });
+
+    let timeoutHandle;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
+    });
+
+    const result = await Promise.race([
+      runningJobPromise.then(() => 'completed').catch(() => 'completed'),
+      timeoutPromise
+    ]);
+
+    clearTimeout(timeoutHandle);
+
+    if (result === 'timeout') {
+      logger.warn('Shutdown wait for running job timed out', {
+        jobId: currentJobId,
+        timeout: config.SHUTDOWN_TIMEOUT
+      });
+    }
+  }
+
+  // Force release the lock
+  releaseLock();
+
+  logger.info('Shutdown complete');
+}
+
+/**
+ * Register process signal handlers for graceful shutdown
+ * Listens for SIGTERM and SIGINT signals
+ */
+function registerSignalHandlers() {
+  process.on('SIGTERM', () => {
+    gracefulShutdown();
+  });
+
+  process.on('SIGINT', () => {
+    gracefulShutdown();
+  });
+}
+
+module.exports = {
+  isScraperRunning,
+  getCurrentJobId,
+  acquireLock,
+  releaseLock,
+  getScheduleExpression,
+  getNextScheduledRun,
+  initializeScheduler,
+  stopScheduler,
+  isShuttingDown,
+  gracefulShutdown,
+  registerSignalHandlers
+};

middleware/auth.js

@@ -0,0 +1,111 @@
+const jwt = require('jsonwebtoken');
+const authConfig = require('../config/auth');
+const { findById } = require('../models/user');
+
+/**
+ * Check if a token should be refreshed based on its age
+ * @param {Object} decoded - Decoded JWT payload
+ * @returns {boolean} True if token should be refreshed
+ */
+const shouldRefreshToken = (decoded) => {
+  const tokenAge = Math.floor(Date.now() / 1000) - decoded.iat;
+  return tokenAge > authConfig.jwt.refreshThreshold;
+};
+
+/**
+ * Generate a new JWT token for a user
+ * @param {string|ObjectId} userId - User's MongoDB _id
+ * @returns {string} JWT token
+ */
+const generateToken = (userId) => {
+  return jwt.sign(
+    { userId: userId.toString() },
+    authConfig.jwt.secret,
+    { expiresIn: authConfig.jwt.expiresIn }
+  );
+};
+
+/**
+ * Authentication middleware
+ * Validates JWT token from cookie and attaches user to request
+ * Implements sliding window token refresh
+ *
+ * @param {Request} req - Express request object
+ * @param {Response} res - Express response object
+ * @param {Function} next - Express next function
+ */
+const requireAuth = async (req, res, next) => {
+  const token = req.cookies[authConfig.cookie.name];
+
+  // No token present
+  if (!token) {
+    return res.status(401).json({ error: 'Authentication required' });
+  }
+
+  try {
+    // Verify the token
+    const decoded = jwt.verify(token, authConfig.jwt.secret);
+
+    // Get database instance
+    const db = req.app.locals.db;
+
+    // Fetch user from database
+    const user = await findById(db, decoded.userId);
+
+    // User not found
+    if (!user) {
+      res.clearCookie(authConfig.cookie.name);
+      return res.status(401).json({ error: 'Authentication required' });
+    }
+
+    // User is disabled (silent logout)
+    if (!user.isActive) {
+      res.clearCookie(authConfig.cookie.name);
+      return res.status(401).json({ error: 'Authentication required' });
+    }
+
+    // Attach user to request
+    req.user = user;
+
+    // Sliding window token refresh
+    if (shouldRefreshToken(decoded)) {
+      const newToken = generateToken(user._id);
+      res.cookie(authConfig.cookie.name, newToken, authConfig.cookie.options);
+    }
+
+    next();
+  } catch (err) {
+    // Token verification failed (invalid or expired)
+    res.clearCookie(authConfig.cookie.name);
+    return res.status(401).json({ error: 'Invalid token' });
+  }
+};
+
+/**
+ * Admin authorization middleware
+ * Must be used after requireAuth middleware
+ * Checks if authenticated user has admin role
+ *
+ * @param {Request} req - Express request object
+ * @param {Response} res - Express response object
+ * @param {Function} next - Express next function
+ */
+const requireAdmin = (req, res, next) => {
+  // Check if user is attached (requireAuth should be called first)
+  if (!req.user) {
+    return res.status(401).json({ error: 'Authentication required' });
+  }
+
+  // Check if user has admin role
+  if (req.user.role !== 'admin') {
+    return res.status(403).json({ error: 'Admin access required' });
+  }
+
+  next();
+};
+
+module.exports = {
+  requireAuth,
+  requireAdmin,
+  generateToken
+};

middleware/passport.js

@@ -0,0 +1,36 @@
+const passport = require('passport');
+const GoogleStrategy = require('passport-google-oauth20').Strategy;
+const authConfig = require('../config/auth');
+const { findOrCreateUser } = require('../models/user');
+
+const configurePassport = (passport, db) => {
+  passport.use(new GoogleStrategy({
+    clientID: authConfig.google.clientID,
+    clientSecret: authConfig.google.clientSecret,
+    callbackURL: authConfig.google.callbackURL
+  },
+  async (accessToken, refreshToken, profile, done) => {
+    try {
+      // Safely extract email
+      const email = profile.emails?.[0]?.value;
+      if (!email) {
+        return done(new Error('No email found in Google profile'), null);
+      }
+
+      const userProfile = {
+        googleId: profile.id,
+        email: email,
+        name: profile.displayName,
+        picture: profile.photos?.[0]?.value || null
+      };
+
+      const user = await findOrCreateUser(db, userProfile);
+      done(null, user);
+    } catch (error) {
+      console.error('Passport strategy error:', error);
+      done(error, null);
+    }
+  }));
+};
+
+module.exports = { configurePassport };

models/user.js

@@ -0,0 +1,253 @@
+const { ObjectId } = require('mongodb');
+
+const USER_COLLECTION = 'users';
+
+/**
+ * Find or create a user based on Google OAuth profile
+ * Uses upsert pattern to handle both new and returning users
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} profile - Google OAuth profile
+ * @param {string} profile.id - Google's unique user ID
+ * @param {string} profile.displayName - User's display name
+ * @param {Array} profile.emails - Array of email objects
+ * @param {Array} profile.photos - Array of photo objects
+ * @returns {Promise<Object>} User document
+ */
+async function findOrCreateUser(db, profile) {
+  const now = new Date();
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { googleId: profile.googleId },
+    {
+      $set: {
+        email: profile.email,
+        name: profile.name,
+        picture: profile.picture || null,
+        lastLoginAt: now
+      },
+      $inc: { loginCount: 1 },
+      $setOnInsert: {
+        googleId: profile.googleId,
+        isActive: true,
+        role: 'user',
+        createdAt: now
+      }
+    },
+    {
+      upsert: true,
+      returnDocument: 'after'
+    }
+  );
+
+  return result;
+}
+
+/**
+ * Find a user by Google ID
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string} googleId - Google's unique user ID
+ * @returns {Promise<Object|null>} User document or null
+ */
+async function findByGoogleId(db, googleId) {
+  return await db.collection(USER_COLLECTION).findOne({ googleId });
+}
+
+/**
+ * Find a user by MongoDB ObjectId
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @returns {Promise<Object|null>} User document or null
+ */
+async function findById(db, id) {
+  // Convert string to ObjectId if needed
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+  return await db.collection(USER_COLLECTION).findOne({ _id: objectId });
+}
+
+/**
+ * Enable or disable a user account
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @param {boolean} isActive - New active status
+ * @param {string|ObjectId|null} adminId - Admin performing the action (required when disabling)
+ * @returns {Promise<Object>} Update result
+ */
+async function setUserActive(db, id, isActive, adminId = null) {
+  // Convert string to ObjectId if needed
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+
+  let updateDoc;
+  if (isActive) {
+    // Enabling: clear disabledAt and disabledBy
+    updateDoc = {
+      $set: { isActive: true, disabledAt: null, disabledBy: null }
+    };
+  } else {
+    // Disabling: set disabledAt and disabledBy
+    // Convert string to ObjectId if needed, keep ObjectId as-is
+    const adminObjectId = adminId && typeof adminId === 'string'
+      ? new ObjectId(adminId)
+      : adminId;
+    updateDoc = {
+      $set: {
+        isActive: false,
+        disabledAt: new Date(),
+        disabledBy: adminObjectId
+      }
+    };
+  }
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { _id: objectId },
+    updateDoc,
+    { returnDocument: 'after' }
+  );
+
+  return result;
+}
+
+/**
+ * Create required indexes for the users collection
+ * Should be called once during application startup
+ *
+ * @param {Db} db - MongoDB database instance
+ * @returns {Promise<void>}
+ */
+async function createIndexes(db) {
+  const collection = db.collection(USER_COLLECTION);
+
+  // Unique index on googleId for OAuth lookups
+  await collection.createIndex(
+    { googleId: 1 },
+    { unique: true }
+  );
+
+  // Unique index on email for user identification
+  await collection.createIndex(
+    { email: 1 },
+    { unique: true }
+  );
+
+  // Compound index for querying active users sorted by last login
+  await collection.createIndex(
+    { isActive: 1, lastLoginAt: -1 }
+  );
+
+  // Index on role for admin queries
+  await collection.createIndex({ role: 1 });
+
+  // Compound index on isActive and role for filtered admin queries
+  await collection.createIndex({ isActive: 1, role: 1 });
+
+  // Index on createdAt for recent registrations
+  await collection.createIndex({ createdAt: -1 });
+
+  // Index on lastLoginAt for recently active users
+  await collection.createIndex({ lastLoginAt: -1 });
+
+  console.log('User collection indexes created successfully');
+}
+
+/**
+ * Get paginated list of users with optional filtering and sorting
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} options - Query options
+ * @param {number} options.page - Page number (1-indexed)
+ * @param {number} options.limit - Items per page (max 100)
+ * @param {string} options.search - Search term for name/email
+ * @param {string} options.status - Filter by status: 'all', 'active', 'disabled'
+ * @param {string} options.sort - Sort field: 'createdAt', 'lastLoginAt', 'loginCount'
+ * @param {string} options.order - Sort order: 'asc', 'desc'
+ * @returns {Promise<{users: Array, pagination: Object}>}
+ */
+async function getPaginatedUsers(db, options = {}) {
+  const page = Math.max(1, parseInt(options.page) || 1);
+  const limit = Math.min(100, Math.max(1, parseInt(options.limit) || 20));
+  const skip = (page - 1) * limit;
+
+  // Build filter
+  const filter = {};
+  if (options.search) {
+    const searchRegex = new RegExp(options.search, 'i');
+    filter.$or = [{ name: searchRegex }, { email: searchRegex }];
+  }
+  if (options.status === 'active') {
+    filter.isActive = true;
+  } else if (options.status === 'disabled') {
+    filter.isActive = false;
+  }
+
+  // Build sort
+  const sortField = ['createdAt', 'lastLoginAt', 'loginCount'].includes(options.sort)
+    ? options.sort
+    : 'createdAt';
+  const sortOrder = options.order === 'asc' ? 1 : -1;
+  const sort = { [sortField]: sortOrder };
+
+  const collection = db.collection(USER_COLLECTION);
+
+  // Execute queries in parallel
+  const [users, total] = await Promise.all([
+    collection.find(filter).sort(sort).skip(skip).limit(limit).toArray(),
+    collection.countDocuments(filter)
+  ]);
+
+  return {
+    users,
+    pagination: {
+      page,
+      limit,
+      total,
+      pages: Math.ceil(total / limit)
+    }
+  };
+}
+
+/**
+ * Update a user's role
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @param {string} role - New role ('user' or 'admin')
+ * @returns {Promise<Object|null>} Updated user or null
+ */
+async function updateUserRole(db, id, role) {
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { _id: objectId },
+    { $set: { role } },
+    { returnDocument: 'after' }
+  );
+
+  return result;
+}
+
+/**
+ * Check if a string is a valid MongoDB ObjectId
+ *
+ * @param {string} id - String to validate
+ * @returns {boolean} True if valid ObjectId format
+ */
+function isValidObjectId(id) {
+  if (typeof id !== 'string') return false;
+  if (!id || !id.trim()) return false;
+  return /^[0-9a-fA-F]{24}$/.test(id);
+}
+
+module.exports = {
+  USER_COLLECTION,
+  findOrCreateUser,
+  findByGoogleId,
+  findById,
+  setUserActive,
+  createIndexes,
+  getPaginatedUsers,
+  updateUserRole,
+  isValidObjectId
+};

package.json

@@ -4,28 +4,47 @@
     "description": "API backend for Country Club Towers & Gardens apartment dashboard",
     "main": "server.js",
     "scripts": {
-      "start": "node server.js",
+        "start": "node server.js",
-      "dev": "nodemon server.js",
+        "dev": "nodemon server.js",
-      "test": "echo \"Error: no test specified\" && exit 1"
+        "test": "jest",
+        "test:watch": "jest --watch",
+        "test:coverage": "jest --coverage",
+        "lint": "eslint .",
+        "lint:fix": "eslint . --fix"
     },
     "keywords": [
-      "apartments",
+        "apartments",
-      "api",
+        "api",
-      "real-estate",
+        "real-estate",
-      "monitoring"
+        "monitoring"
     ],
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
-      "express": "^4.18.2",
+        "axios": "^1.13.4",
-      "mongodb": "^6.3.0",
+        "cheerio": "^1.2.0",
-      "cors": "^2.8.5",
+        "cookie-parser": "^1.4.7",
-      "express-rate-limit": "^7.1.5"
+        "cors": "^2.8.5",
+        "cron-parser": "^5.5.0",
+        "express": "^4.18.2",
+        "express-rate-limit": "^7.1.5",
+        "jsonwebtoken": "^9.0.3",
+        "mongodb": "^6.3.0",
+        "node-cron": "^4.2.1",
+        "passport": "^0.7.0",
+        "passport-google-oauth20": "^2.0.0",
+        "uuid": "^13.0.0"
     },
     "devDependencies": {
-      "nodemon": "^3.0.2"
+        "@eslint/js": "^9.18.0",
+        "eslint": "^9.18.0",
+        "globals": "^15.14.0",
+        "jest": "^30.2.0",
+        "mongodb-memory-server": "^11.0.1",
+        "nodemon": "^3.0.2",
+        "supertest": "^7.2.2"
     },
     "engines": {
-      "node": ">=18.0.0"
+        "node": ">=18.0.0"
     }
-  }
+}

routes/activity.js

@@ -0,0 +1,179 @@
+const express = require('express');
+const { ObjectId } = require('mongodb');
+const { requireAuth, requireAdmin } = require('../middleware/auth');
+const { logActivity, ACTIONS, ACTIVITY_COLLECTION } = require('../services/activityLogger');
+
+const router = express.Router();
+
+/**
+ * POST /log - Log frontend activity
+ * Protected with requireAuth
+ */
+router.post('/log', requireAuth, async (req, res) => {
+  try {
+    const { action, metadata } = req.body;
+
+    // Validate action is in ACTIONS object
+    if (!action || !Object.values(ACTIONS).includes(action)) {
+      return res.status(400).json({ error: 'Invalid action type' });
+    }
+
+    const db = req.app.locals.db;
+
+    // Log the activity with merged metadata
+    await logActivity(
+      db,
+      req.user._id,
+      action,
+      { ...metadata, page: metadata?.page },
+      req
+    );
+
+    res.json({ success: true });
+  } catch (error) {
+    console.error('Error logging activity:', error);
+    res.status(500).json({ error: 'Failed to log activity' });
+  }
+});
+
+/**
+ * GET /user/:userId - Get activity for specific user
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/user/:userId', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const { userId } = req.params;
+    const limit = parseInt(req.query.limit) || 50;
+    const skip = parseInt(req.query.skip) || 0;
+
+    const db = req.app.locals.db;
+
+    // Convert userId to ObjectId
+    let userObjectId;
+    try {
+      userObjectId = new ObjectId(userId);
+    } catch (error) {
+      return res.status(400).json({ error: 'Invalid user ID format' });
+    }
+
+    // Find activities for the user
+    const activities = await db
+      .collection(ACTIVITY_COLLECTION)
+      .find({ userId: userObjectId })
+      .sort({ timestamp: -1 })
+      .skip(skip)
+      .limit(limit)
+      .toArray();
+
+    // Get total count
+    const total = await db
+      .collection(ACTIVITY_COLLECTION)
+      .countDocuments({ userId: userObjectId });
+
+    res.json({ activities, total });
+  } catch (error) {
+    console.error('Error fetching user activity:', error);
+    res.status(500).json({ error: 'Failed to fetch user activity' });
+  }
+});
+
+/**
+ * GET /recent - Get recent activity across all users
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/recent', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const limit = parseInt(req.query.limit) || 50;
+    const skip = parseInt(req.query.skip) || 0;
+
+    const db = req.app.locals.db;
+
+    // Find all activities sorted by timestamp
+    const activities = await db
+      .collection(ACTIVITY_COLLECTION)
+      .find({})
+      .sort({ timestamp: -1 })
+      .skip(skip)
+      .limit(limit)
+      .toArray();
+
+    // Get total count
+    const total = await db.collection(ACTIVITY_COLLECTION).countDocuments({});
+
+    res.json({ activities, total });
+  } catch (error) {
+    console.error('Error fetching recent activity:', error);
+    res.status(500).json({ error: 'Failed to fetch recent activity' });
+  }
+});
+
+/**
+ * GET /stats - Get activity statistics
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/stats', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+
+    // Calculate date 7 days ago
+    const sevenDaysAgo = new Date();
+    sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+
+    // Aggregate activity counts by action type in last 7 days
+    const actionCountsResult = await db
+      .collection(ACTIVITY_COLLECTION)
+      .aggregate([
+        {
+          $match: {
+            timestamp: { $gte: sevenDaysAgo }
+          }
+        },
+        {
+          $group: {
+            _id: '$action',
+            count: { $sum: 1 }
+          }
+        }
+      ])
+      .toArray();
+
+    // Convert array to object
+    const actionCounts = {};
+    actionCountsResult.forEach(item => {
+      actionCounts[item._id] = item.count;
+    });
+
+    // Count unique active users in last 7 days
+    const activeUsersResult = await db
+      .collection(ACTIVITY_COLLECTION)
+      .aggregate([
+        {
+          $match: {
+            timestamp: { $gte: sevenDaysAgo }
+          }
+        },
+        {
+          $group: {
+            _id: '$userId'
+          }
+        },
+        {
+          $count: 'total'
+        }
+      ])
+      .toArray();
+
+    const activeUsers = activeUsersResult.length > 0 ? activeUsersResult[0].total : 0;
+
+    res.json({
+      actionCounts,
+      activeUsers,
+      period: '7d'
+    });
+  } catch (error) {
+    console.error('Error fetching activity stats:', error);
+    res.status(500).json({ error: 'Failed to fetch activity statistics' });
+  }
+});
+
+module.exports = router;

routes/auth.js

@@ -0,0 +1,154 @@
+const express = require('express');
+const passport = require('passport');
+const jwt = require('jsonwebtoken');
+const crypto = require('crypto');
+const authConfig = require('../config/auth');
+const { requireAuth, generateToken } = require('../middleware/auth');
+const { logActivity, ACTIONS } = require('../services/activityLogger');
+
+const router = express.Router();
+
+/**
+ * GET /auth/google
+ * Initiates Google OAuth flow with state parameter for CSRF protection
+ */
+router.get('/google', (req, res, next) => {
+  // Generate cryptographically secure state parameter
+  const state = crypto.randomBytes(32).toString('hex');
+
+  // Store state in a short-lived cookie for validation
+  res.cookie('oauth_state', state, {
+    httpOnly: true,
+    secure: process.env.NODE_ENV === 'production',
+    sameSite: 'lax',
+    maxAge: 5 * 60 * 1000 // 5 minutes
+  });
+
+  passport.authenticate('google', {
+    scope: authConfig.google.scope,
+    session: false,
+    state: state
+  })(req, res, next);
+});
+
+/**
+ * GET /auth/google/callback
+ * Handles OAuth callback from Google
+ * Validates state parameter for CSRF protection
+ * On success: generates JWT, sets cookie, redirects to frontend
+ * On failure: redirects to login with error
+ */
+router.get('/google/callback', (req, res, next) => {
+  // Validate state parameter to prevent CSRF
+  const stateFromCookie = req.cookies.oauth_state;
+  const stateFromQuery = req.query.state;
+
+  // Clear the state cookie immediately
+  res.clearCookie('oauth_state');
+
+  if (!stateFromCookie || !stateFromQuery || stateFromCookie !== stateFromQuery) {
+    console.warn('OAuth state mismatch - potential CSRF attack');
+    return res.redirect(`${process.env.FRONTEND_URL}/login?error=invalid_state`);
+  }
+
+  // State is valid, proceed with authentication
+  passport.authenticate('google', {
+    session: false,
+    failureRedirect: `${process.env.FRONTEND_URL}/login?error=auth_failed`
+  })(req, res, next);
+}, async (req, res) => {
+    try {
+      const user = req.user;
+
+      // Check if email is verified (if available in profile)
+      if (req.authInfo && req.authInfo.emails && req.authInfo.emails[0]) {
+        const emailVerified = req.authInfo.emails[0].verified !== false; // Default to true if not present
+        if (!emailVerified) {
+          return res.redirect(`${process.env.FRONTEND_URL}/login?error=unverified`);
+        }
+      }
+
+      // Check if user account is active
+      if (!user.isActive) {
+        return res.redirect(`${process.env.FRONTEND_URL}/login`);
+      }
+
+      // Generate JWT token
+      const token = generateToken(user._id);
+
+      // Set auth cookie
+      res.cookie(authConfig.cookie.name, token, authConfig.cookie.options);
+
+      // Log login activity
+      const db = req.app.locals.db;
+      await logActivity(db, user._id, ACTIONS.LOGIN, { method: 'google' }, req);
+
+      // Redirect to frontend
+      res.redirect(process.env.FRONTEND_URL);
+    } catch (err) {
+      console.error('OAuth callback error:', err);
+      res.redirect(`${process.env.FRONTEND_URL}/login?error=auth_failed`);
+    }
+  }
+);
+
+/**
+ * GET /auth/me
+ * Returns current authenticated user's data
+ * Protected route - requires valid JWT
+ */
+router.get('/me', requireAuth, (req, res) => {
+  res.json({
+    user: {
+      id: req.user._id,
+      email: req.user.email,
+      name: req.user.name,
+      picture: req.user.picture,
+      role: req.user.role
+    }
+  });
+});
+
+/**
+ * POST /auth/logout
+ * Clears authentication cookie
+ * Protected route - requires valid JWT
+ */
+router.post('/logout', requireAuth, async (req, res) => {
+  // Log logout activity before clearing cookie
+  const db = req.app.locals.db;
+  await logActivity(db, req.user._id, ACTIONS.LOGOUT, {}, req);
+
+  // Clear the auth cookie
+  res.clearCookie(authConfig.cookie.name, {
+    ...authConfig.cookie.options,
+    maxAge: 0
+  });
+
+  res.json({ message: 'Logged out successfully' });
+});
+
+/**
+ * GET /auth/status
+ * Returns authentication status without requiring middleware
+ * Used for quick frontend checks
+ */
+router.get('/status', (req, res) => {
+  const token = req.cookies[authConfig.cookie.name];
+
+  // No token present
+  if (!token) {
+    return res.json({ authenticated: false });
+  }
+
+  try {
+    // Verify the token
+    jwt.verify(token, authConfig.jwt.secret);
+    return res.json({ authenticated: true });
+  } catch (err) {
+    // Token invalid or expired
+    return res.json({ authenticated: false });
+  }
+});
+
+module.exports = router;

server.js

@@ -2,12 +2,36 @@ const express = require('express');
 const { MongoClient } = require('mongodb');
 const cors = require('cors');
 const rateLimit = require('express-rate-limit');
+const cookieParser = require('cookie-parser');
+const passport = require('passport');
+const { configurePassport } = require('./middleware/passport');
+const { requireAuth } = require('./middleware/auth');
+const authRoutes = require('./routes/auth');
+const activityRoutes = require('./routes/activity');
+const adminRoutes = require('./routes/admin');
+const { createIndexes } = require('./models/user');
+const { createActivityIndexes } = require('./services/activityLogger');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
 
 // Configuration
-const MONGO_URI = process.env.MONGO_URI || "mongodb://admin:password123@localhost:27017";
+if (!process.env.MONGO_URI && process.env.NODE_ENV === 'production') {
+  console.error('❌ MONGO_URI environment variable is required in production');
+  process.exit(1);
+}
+const MONGO_URI = process.env.MONGO_URI || "mongodb://localhost:27017";
+
+// Log environment configuration status (safe - no secret values)
+console.log('📋 Environment Configuration:');
+console.log(`   NODE_ENV: ${process.env.NODE_ENV || 'development'}`);
+console.log(`   MONGO_URI: ${MONGO_URI ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CLIENT_ID: ${process.env.GOOGLE_CLIENT_ID ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CLIENT_SECRET: ${process.env.GOOGLE_CLIENT_SECRET ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CALLBACK_URL: ${process.env.GOOGLE_CALLBACK_URL || '✗ Missing'}`);
+console.log(`   JWT_SECRET: ${process.env.JWT_SECRET ? `✓ Set (${process.env.JWT_SECRET.length} chars)` : '✗ Missing'}`);
+console.log(`   FRONTEND_URL: ${process.env.FRONTEND_URL || 'http://localhost:5173 (default)'}`);
+console.log(`   ACTIVITY_LOG_LEVEL: ${process.env.ACTIVITY_LOG_LEVEL || 'all (default)'}`);
 const DB_NAME = "apartments";
 const UNITS_COLLECTION = "units_migration_test";
 const PRICES_COLLECTION = "unit_prices_migration_test";
@@ -16,7 +40,17 @@ const DAILY_SUMMARIES_COLLECTION = "daily_summaries";
 console.log(`🔗 Connecting to MongoDB: ${MONGO_URI.includes('localhost') ? 'Local MongoDB' : 'Atlas MongoDB'}`);
 
 // Middleware
-app.use(cors());
+const FRONTEND_URL = process.env.FRONTEND_URL || 'http://localhost:5173';
+
+const corsOptions = {
+  origin: FRONTEND_URL,
+  credentials: true,
+  methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
+  allowedHeaders: ['Content-Type', 'Authorization'],
+  exposedHeaders: ['set-cookie']
+};
+app.use(cors(corsOptions));
+app.use(cookieParser());
 app.use(express.json());
 
 // Rate limiting
@@ -26,6 +60,9 @@ const limiter = rateLimit({
 });
 app.use(limiter);
 
+// Initialize Passport
+app.use(passport.initialize());
+
 // MongoDB connection
 let db;
 let client;
@@ -35,23 +72,84 @@ async function connectToMongoDB() {
     client = new MongoClient(MONGO_URI);
     await client.connect();
     db = client.db(DB_NAME);
+    app.locals.db = db;
     console.log('✅ Successfully connected to MongoDB');
+
+    // Configure Passport and create indexes after DB connection
+    configurePassport(passport, db);
+    await createIndexes(db);
+    await createActivityIndexes(db);
+    console.log('✅ Passport configured and indexes created');
   } catch (error) {
     console.error('❌ Failed to connect to MongoDB:', error);
     process.exit(1);
   }
 }
 
-// Helper function to get today's date
+// Helper function to get today's date in UTC
-const getTodayDate = () => new Date().toISOString().split('T')[0];
+const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
 
-// Helper function to get yesterday's date
+// Cache for the most recent date with data
-const getYesterdayDate = () => {
+let cachedLatestDate = null;
-  const yesterday = new Date();
+let cachedLatestDateTimestamp = 0;
-  yesterday.setDate(yesterday.getDate() - 1);
+const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
-  return yesterday.toISOString().split('T')[0];
+
+// Helper function to get the most recent date with data in the database
+// This handles timezone mismatches between server and data collection
+const getLatestDateWithData = async () => {
+  const now = Date.now();
+
+  // Return cached value if still valid
+  if (cachedLatestDate && (now - cachedLatestDateTimestamp) < CACHE_TTL) {
+    return cachedLatestDate;
+  }
+
+  try {
+    const result = await db.collection(PRICES_COLLECTION)
+      .find({})
+      .sort({ date_checked: -1 })
+      .limit(1)
+      .project({ date_checked: 1 })
+      .toArray();
+
+    if (result.length > 0) {
+      cachedLatestDate = result[0].date_checked;
+      cachedLatestDateTimestamp = now;
+      return cachedLatestDate;
+    }
+  } catch (error) {
+    console.error('Error fetching latest date with data:', error);
+  }
+
+  // Fallback to UTC today if no data found
+  return getTodayDateUTC();
 };
 
+// Helper function to get yesterday relative to the latest date with data
+const getYesterdayDate = (today) => {
+  const todayDate = new Date(today + 'T00:00:00Z');
+  todayDate.setDate(todayDate.getDate() - 1);
+  return todayDate.toISOString().split('T')[0];
+};
+
+// Helper function to validate unit code (prevents NoSQL injection)
+const isValidUnitCode = (unitCode) => {
+  // Allow alphanumeric characters, hyphens, and underscores, max 20 chars
+  return typeof unitCode === 'string' &&
+         unitCode.length > 0 &&
+         unitCode.length <= 20 &&
+         /^[A-Za-z0-9_-]+$/.test(unitCode);
+};
+
+// Mount auth routes
+app.use('/auth', authRoutes);
+
+// Mount activity routes
+app.use('/activity', activityRoutes);
+
+// Mount admin routes (Traefik strips /api prefix, so /api/admin becomes /admin)
+app.use('/admin', adminRoutes);
+
 // Health check endpoint
 app.get('/health', (req, res) => {
   res.json({
@@ -62,7 +160,7 @@ app.get('/health', (req, res) => {
 });
 
 // Get last scrape time
-app.get('/last-scrape', async (req, res) => {
+app.get('/last-scrape', requireAuth, async (req, res) => {
   try {
     // Get the most recent price record using _id (ObjectId contains timestamp)
     const lastRecord = await db.collection(PRICES_COLLECTION)
@@ -94,9 +192,9 @@ app.get('/last-scrape', async (req, res) => {
 });
 
 // Get daily summary (matches your daily_summaries collection)
-app.get('/daily-summary', async (req, res) => {
+app.get('/daily-summary', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     
     // Try to get today's summary first
     let summary = await db.collection(DAILY_SUMMARIES_COLLECTION)
@@ -133,14 +231,16 @@ app.get('/daily-summary', async (req, res) => {
 });
 
 // Get price history (last 15 days of average prices)
-app.get('/price-history', async (req, res) => {
+app.get('/price-history', requireAuth, async (req, res) => {
   try {
     const days = parseInt(req.query.days) || 15;
+    const latestDate = await getLatestDateWithData();
 
-    // Generate date range
+    // Generate date range ending at the latest date with data
     const dates = [];
+    const baseDate = new Date(latestDate + 'T00:00:00Z');
     for (let i = days - 1; i >= 0; i--) {
-      const date = new Date();
+      const date = new Date(baseDate);
       date.setDate(date.getDate() - i);
       dates.push(date.toISOString().split('T')[0]);
     }
@@ -188,9 +288,10 @@ app.get('/price-history', async (req, res) => {
 });
 
 // Get available units with current prices
-app.get('/available-units', async (req, res) => {
+app.get('/available-units', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
+    const yesterday = getYesterdayDate(today);
 
     // Get units that have prices today (excluding furnished units)
     const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
@@ -215,11 +316,17 @@ app.get('/available-units', async (req, res) => {
       },
       {
         $addFields: {
-          currentPrice: {
+          currentPriceDoc: {
             $arrayElemAt: [
               { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", today] } } },
               0
             ]
+          },
+          yesterdayPriceDoc: {
+            $arrayElemAt: [
+              { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", yesterday] } } },
+              0
+            ]
           }
         }
       },
@@ -273,26 +380,52 @@ app.get('/available-units', async (req, res) => {
                 }
               }
             }
+          }
+        }
+      },
+      {
+        $addFields: {
+          // Filter prices to only include those from availability start date onwards
+          availabilityPrices: {
+            $filter: {
+              input: "$all_prices",
+              cond: { $gte: ["$$this.date_checked", "$availabilityCalc.startDate"] }
+            }
           },
+          priceHistory: {
+            $slice: [
+              { $sortArray: { input: "$all_prices", sortBy: { date_checked: -1 } } },
+              30
+            ]
+          }
+        }
+      },
+      {
+        $addFields: {
           priceStats: {
             $let: {
               vars: {
-                prices: { $map: { input: "$all_prices", as: "p", in: "$$p.price" } }
+                prices: { $map: { input: "$availabilityPrices", as: "p", in: "$$p.price" } }
               },
               in: {
                 minPrice: { $min: "$$prices" },
-                maxPrice: { $max: "$$prices" },
+                maxPrice: { $max: "$$prices" }
-                priceHistory: {
-                  $slice: [
-                    { $sortArray: { input: "$all_prices", sortBy: { date_checked: -1 } } },
-                    30
-                  ]
-                }
               }
             }
           }
         }
       },
+      {
+        $addFields: {
+          priceChangeToday: {
+            $cond: {
+              if: { $and: [{ $ne: ["$currentPriceDoc.price", null] }, { $ne: ["$yesterdayPriceDoc.price", null] }] },
+              then: { $subtract: ["$currentPriceDoc.price", "$yesterdayPriceDoc.price"] },
+              else: null
+            }
+          }
+        }
+      },
       {
         $project: {
           unitCode: "$unit_code",
@@ -303,7 +436,9 @@ app.get('/available-units', async (req, res) => {
           community: "$community",
           available: "$available",
           soonest: "$soonest",
-          currentPrice: "$currentPrice.price",
+          currentPrice: "$currentPriceDoc.price",
+          yesterdayPrice: "$yesterdayPriceDoc.price",
+          priceChangeToday: { $ifNull: ["$priceChangeToday", null] },
           minPrice: "$priceStats.minPrice",
           maxPrice: "$priceStats.maxPrice",
           daysAvailable: {
@@ -315,7 +450,7 @@ app.get('/available-units', async (req, res) => {
           },
           priceHistory: {
             $map: {
-              input: "$priceStats.priceHistory",
+              input: "$priceHistory",
               as: "ph",
               in: {
                 date: "$$ph.date_checked",
@@ -336,10 +471,15 @@ app.get('/available-units', async (req, res) => {
 });
 
 // Get full price history for a specific unit
-app.get('/unit/:unitCode/price-history', async (req, res) => {
+app.get('/unit/:unitCode/price-history', requireAuth, async (req, res) => {
   try {
     const { unitCode } = req.params;
 
+    // Validate unitCode to prevent NoSQL injection
+    if (!isValidUnitCode(unitCode)) {
+      return res.status(400).json({ error: 'Invalid unit code format' });
+    }
+
     const priceHistory = await db.collection(PRICES_COLLECTION)
       .find({ unit_code: unitCode })
       .sort({ date_checked: 1 })
@@ -363,9 +503,9 @@ app.get('/unit/:unitCode/price-history', async (req, res) => {
 });
 
 // Get comprehensive analytics data
-app.get('/analytics', async (req, res) => {
+app.get('/analytics', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
 
     // 1. Monthly Price Trends - aggregate all prices by month
     const monthlyTrends = await db.collection(PRICES_COLLECTION).aggregate([
@@ -605,10 +745,10 @@ app.get('/analytics', async (req, res) => {
 });
 
 // Get recent activity (new units, rented units, price changes)
-app.get('/recent-activity', async (req, res) => {
+app.get('/recent-activity', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
-    const yesterday = getYesterdayDate();
+    const yesterday = getYesterdayDate(today);
     
     // Get today's summary for new/rented units
     const todaySummary = await db.collection(DAILY_SUMMARIES_COLLECTION)
@@ -734,9 +874,9 @@ app.get('/recent-activity', async (req, res) => {
 });
 
 // Get plan statistics
-app.get('/plan-stats', async (req, res) => {
+app.get('/plan-stats', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     
     const planStats = await db.collection(UNITS_COLLECTION).aggregate([
       {
@@ -797,9 +937,9 @@ app.get('/plan-stats', async (req, res) => {
 
 // Expanded api
 // Get best deals (units priced below their historical average)
-app.get('/best-deals', async (req, res) => {
+app.get('/best-deals', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const limit = parseInt(req.query.limit) || 5;
 
     const bestDeals = await db.collection(UNITS_COLLECTION).aggregate([
@@ -922,16 +1062,17 @@ app.get('/best-deals', async (req, res) => {
 });
 
 // Get price drops (units with recent price decreases)
-app.get('/price-drops', async (req, res) => {
+app.get('/price-drops', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const daysBack = parseInt(req.query.days) || 7;
     const limit = parseInt(req.query.limit) || 10;
 
-    // Generate date range for the last N days
+    // Generate date range for the last N days relative to latest data date
     const dates = [];
+    const baseDate = new Date(today + 'T00:00:00Z');
     for (let i = 0; i < daysBack; i++) {
-      const date = new Date();
+      const date = new Date(baseDate);
       date.setDate(date.getDate() - i);
       dates.push(date.toISOString().split('T')[0]);
     }
@@ -1019,9 +1160,9 @@ app.get('/price-drops', async (req, res) => {
 });
 
 // Get stale inventory (units on market for a long time)
-app.get('/stale-inventory', async (req, res) => {
+app.get('/stale-inventory', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const minDays = parseInt(req.query.minDays) || 10;
     const limit = parseInt(req.query.limit) || 10;
 
@@ -1145,9 +1286,9 @@ app.get('/stale-inventory', async (req, res) => {
 });
 
 // Get market insights and predictions
-app.get('/market-insights', async (req, res) => {
+app.get('/market-insights', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
 
     // Get various market metrics
     const [
@@ -1355,10 +1496,10 @@ app.get('/market-insights', async (req, res) => {
 });
 
 // Enhanced available units with more details
-app.get('/available-units-enhanced', async (req, res) => {
+app.get('/available-units-enhanced', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
-    const yesterday = getYesterdayDate();
+    const yesterday = getYesterdayDate(today);
 
     const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
       {
@@ -1541,10 +1682,16 @@ app.get('/available-units-enhanced', async (req, res) => {
 });
 
 // Get unit details by unit code
-app.get('/unit/:unitCode', async (req, res) => {
+app.get('/unit/:unitCode', requireAuth, async (req, res) => {
   try {
     const { unitCode } = req.params;
-    const today = getTodayDate();
+
+    // Validate unitCode to prevent NoSQL injection
+    if (!isValidUnitCode(unitCode)) {
+      return res.status(400).json({ error: 'Invalid unit code format' });
+    }
+
+    const today = await getLatestDateWithData();
 
     const unit = await db.collection(UNITS_COLLECTION).aggregate([
       { $match: { unit_code: unitCode } },

services/activityLogger.js

@@ -0,0 +1,192 @@
+const { ObjectId } = require('mongodb');
+
+// Collection name
+const ACTIVITY_COLLECTION = 'user_activity';
+
+// Activity action constants
+const ACTIONS = {
+  LOGIN: 'login',
+  LOGOUT: 'logout',
+  PAGE_VIEW: 'page_view',
+  NAVIGATION: 'navigation',
+  UNIT_VIEW: 'unit_view',
+  UNIT_WATCH: 'unit_watch',
+  FILTER_CHANGE: 'filter_change',
+  SORT_CHANGE: 'sort_change',
+  SEARCH: 'search',
+  EXPORT: 'export',
+  VIEW_TOGGLE: 'view_toggle'
+};
+
+// Log levels define which actions should be logged
+const LOG_LEVELS = {
+  all: [
+    'login',
+    'logout',
+    'page_view',
+    'navigation',
+    'filter_change',
+    'sort_change',
+    'search',
+    'unit_view',
+    'unit_watch',
+    'export',
+    'view_toggle'
+  ],
+  navigation: [
+    'login',
+    'logout',
+    'page_view',
+    'navigation'
+  ],
+  none: []
+};
+
+/**
+ * Get the current activity log level from environment variable
+ * @returns {string} Current log level ('all', 'navigation', or 'none')
+ */
+function getActivityLevel() {
+  const level = process.env.ACTIVITY_LOG_LEVEL || 'all';
+
+  // Validate level exists, default to 'all' if invalid
+  if (!LOG_LEVELS[level]) {
+    console.warn(`Invalid ACTIVITY_LOG_LEVEL: ${level}, defaulting to 'all'`);
+    return 'all';
+  }
+
+  return level;
+}
+
+/**
+ * Check if an action should be logged based on current log level
+ * @param {string} action - Action to check
+ * @returns {boolean} True if action should be logged
+ */
+function shouldLog(action) {
+  const level = getActivityLevel();
+  const allowedActions = LOG_LEVELS[level];
+  return allowedActions.includes(action);
+}
+
+/**
+ * Check if an action is an admin action (always logged regardless of log level)
+ * @param {string} action - Action to check
+ * @returns {boolean} True if admin action
+ */
+function isAdminAction(action) {
+  return action && action.startsWith('ADMIN_');
+}
+
+/**
+ * Log a user activity to the database
+ * Supports two calling conventions:
+ * 1. logActivity(db, userId, action, metadata, req) - positional parameters
+ * 2. logActivity(db, { userId, action, metadata }) - object parameter for admin actions
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|Object} userIdOrOptions - User ID string or options object
+ * @param {string} [action] - Action type (use ACTIONS constants)
+ * @param {Object} [metadata] - Additional action-specific data
+ * @param {Object} [req] - Express request object (for IP and user agent)
+ * @returns {Promise<void>}
+ */
+async function logActivity(db, userIdOrOptions, action, metadata = {}, req = null) {
+  let userId;
+  let actualAction;
+  let actualMetadata;
+  let actualReq;
+
+  // Support object-based call for admin actions
+  if (typeof userIdOrOptions === 'object' && userIdOrOptions !== null) {
+    userId = userIdOrOptions.userId;
+    actualAction = userIdOrOptions.action;
+    actualMetadata = userIdOrOptions.metadata || {};
+    actualReq = userIdOrOptions.req || null;
+  } else {
+    userId = userIdOrOptions;
+    actualAction = action;
+    actualMetadata = metadata;
+    actualReq = req;
+  }
+
+  // Admin actions bypass log level filtering
+  if (!isAdminAction(actualAction) && !shouldLog(actualAction)) {
+    return;
+  }
+
+  try {
+    // Extract IP address (handle proxy forwarding)
+    let ip = null;
+    if (actualReq) {
+      ip = actualReq.ip || actualReq.headers?.['x-forwarded-for']?.split(',')[0] || null;
+    }
+
+    // Extract user agent
+    const userAgent = actualReq?.headers?.['user-agent'] || null;
+
+    // Create activity document
+    const activityDoc = {
+      userId: new ObjectId(userId),
+      action: actualAction,
+      metadata: actualMetadata || {},
+      page: actualMetadata?.page || null,
+      timestamp: new Date(),
+      userAgent: userAgent,
+      ip: ip
+    };
+
+    // Insert into user_activity collection
+    await db.collection(ACTIVITY_COLLECTION).insertOne(activityDoc);
+  } catch (error) {
+    // Log error but don't throw - activity logging should not break the main flow
+    console.error('Error logging activity:', error);
+  }
+}
+
+/**
+ * Create required indexes for the user_activity collection
+ * @param {Db} db - MongoDB database instance
+ * @returns {Promise<void>}
+ */
+async function createActivityIndexes(db) {
+  try {
+    const collection = db.collection(ACTIVITY_COLLECTION);
+
+    // Index for user activity queries (get all activities for a user, sorted by time)
+    await collection.createIndex(
+      { userId: 1, timestamp: -1 },
+      { name: 'userId_timestamp' }
+    );
+
+    // Index for action type queries (get all activities of a certain type)
+    await collection.createIndex(
+      { action: 1, timestamp: -1 },
+      { name: 'action_timestamp' }
+    );
+
+    // TTL index to automatically delete activities older than 90 days
+    await collection.createIndex(
+      { timestamp: 1 },
+      {
+        name: 'timestamp_ttl',
+        expireAfterSeconds: 7776000  // 90 days = 90 * 24 * 60 * 60
+      }
+    );
+
+    console.log('Activity collection indexes created successfully');
+  } catch (error) {
+    console.error('Error creating activity indexes:', error);
+    throw error;
+  }
+}
+
+module.exports = {
+  ACTIVITY_COLLECTION,
+  ACTIONS,
+  LOG_LEVELS,
+  getActivityLevel,
+  shouldLog,
+  logActivity,
+  createActivityIndexes
+};

services/scraperLogger.js

@@ -0,0 +1,164 @@
+/**
+ * Scraper-specific structured JSON logger
+ * Produces one JSON object per line to stdout
+ */
+
+const LEVELS = {
+  info: 'info',
+  warn: 'warn',
+  error: 'error'
+};
+
+/**
+ * Create a logger instance scoped to a job ID
+ * @param {string} jobId - Job identifier for correlation
+ * @returns {Object} Logger object with info, warn, error methods
+ */
+function createLogger(jobId) {
+  /**
+   * Internal log function
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} context - Additional context data
+   */
+  const log = (level, message, context = {}) => {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      message: truncateMessage(message),
+      jobId,
+      context: sanitizeContext(context)
+    };
+
+    // Output as single-line JSON
+    console.log(JSON.stringify(entry));
+  };
+
+  return {
+    info: (message, context) => log(LEVELS.info, message, context),
+    warn: (message, context) => log(LEVELS.warn, message, context),
+    error: (message, context) => log(LEVELS.error, message, context)
+  };
+}
+
+/**
+ * Truncate message to prevent log bloat
+ * @param {string} message - Message to truncate
+ * @param {number} maxLength - Maximum length (default 1000)
+ * @returns {string} Truncated message
+ */
+function truncateMessage(message, maxLength = 1000) {
+  if (message === null || message === undefined) {
+    return '';
+  }
+  const str = String(message);
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.substring(0, maxLength) + '... [truncated]';
+}
+
+/**
+ * Recursively process an object to handle Buffers, circular references, and sensitive data
+ * @param {*} obj - Object to process
+ * @param {WeakSet} seen - Set of seen objects for circular reference detection
+ * @returns {*} Processed value
+ */
+function processValue(obj, seen = new WeakSet()) {
+  // Handle null/undefined
+  if (obj === null) {
+    return null;
+  }
+  if (obj === undefined) {
+    return null;
+  }
+
+  // Handle Buffer BEFORE checking for object (Buffer is an object)
+  if (Buffer.isBuffer(obj)) {
+    return `[Buffer: ${obj.length} bytes]`;
+  }
+
+  // Handle strings - check for MongoDB connection strings
+  if (typeof obj === 'string') {
+    if (/mongodb(\+srv)?:\/\//.test(obj)) {
+      return redactConnectionString(obj);
+    }
+    return obj;
+  }
+
+  // Handle primitives
+  if (typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Handle circular references
+  if (seen.has(obj)) {
+    return '[Circular]';
+  }
+  seen.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => processValue(item, seen));
+  }
+
+  // Handle plain objects
+  const result = {};
+  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
+
+  for (const key of Object.keys(obj)) {
+    // Check for sensitive keys
+    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
+      result[key] = '[REDACTED]';
+    } else {
+      result[key] = processValue(obj[key], seen);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sanitize context object for safe logging
+ * - Remove circular references
+ * - Redact sensitive data
+ * - Handle special types (Buffer, undefined)
+ * @param {Object} context - Context object
+ * @returns {Object} Sanitized context
+ */
+function sanitizeContext(context) {
+  if (!context || typeof context !== 'object') {
+    return {};
+  }
+
+  try {
+    return processValue(context);
+  } catch (error) {
+    // If sanitization fails, return empty context
+    return { sanitizationError: 'Failed to sanitize context' };
+  }
+}
+
+/**
+ * Redact credentials from MongoDB connection string
+ * @param {string} uri - Connection string
+ * @returns {string} Redacted string
+ */
+function redactConnectionString(uri) {
+  try {
+    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
+    return uri.replace(
+      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
+      'mongodb$1://[user]:[REDACTED]@'
+    );
+  } catch {
+    return '[REDACTED CONNECTION STRING]';
+  }
+}
+
+module.exports = {
+  createLogger,
+  LEVELS,
+  // Export internal functions for testing
+  redactConnectionString
+};

services/scraperService.js

@@ -0,0 +1,888 @@
+/**
+ * Scraper Service
+ *
+ * Core scraper logic including HTTP fetching, HTML parsing,
+ * data transformation, and database operations.
+ */
+
+const axios = require('axios');
+const cheerio = require('cheerio');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { createLogger } = require('./scraperLogger');
+
+/**
+ * Sleep utility for retry delays
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Determine if an error is retryable
+ * @param {Error} error - Axios error
+ * @returns {boolean} True if should retry
+ */
+function isRetryableError(error) {
+  // Network errors (timeout, DNS, connection) don't have a response property
+  if (!error.response) {
+    return true;
+  }
+
+  const status = error.response.status;
+
+  // 5xx server errors are retryable
+  if (status >= 500) {
+    return true;
+  }
+
+  // 429 Too Many Requests - do not retry immediately
+  if (status === 429) {
+    return false;
+  }
+
+  // 4xx client errors - do not retry
+  return false;
+}
+
+/**
+ * Fetch page HTML with retry logic
+ * @param {string} url - Target URL
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<string>} HTML content
+ * @throws {Error} After all retries exhausted
+ */
+async function fetchPage(url, logger) {
+  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    try {
+      logger.info('Fetching page', { url, attempt });
+
+      const response = await axios.get(url, {
+        timeout,
+        headers: {
+          'User-Agent': config.USER_AGENT
+        },
+        maxRedirects: 5,
+        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
+      });
+
+      logger.info('Page fetched successfully', {
+        status: response.status,
+        contentLength: response.data.length
+      });
+
+      return response.data;
+
+    } catch (error) {
+      lastError = error;
+
+      // Determine if error is retryable
+      const isRetryable = isRetryableError(error);
+
+      logger.warn('Fetch attempt failed', {
+        attempt,
+        errorType: error.name,
+        errorMessage: error.message,
+        statusCode: error.response?.status,
+        isRetryable
+      });
+
+      // Don't retry non-retryable errors (4xx)
+      if (!isRetryable) {
+        throw error;
+      }
+
+      // Don't wait after last attempt
+      if (attempt <= maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+        logger.info('Waiting before retry', { delay });
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Parse unit data from HTML using cheerio
+ * @param {string} html - HTML content
+ * @param {Object} logger - Logger instance
+ * @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
+ */
+function parseUnits(html, logger) {
+  const $ = cheerio.load(html);
+  const units = [];
+
+  // Find the main container
+  const container = $('section.spaces__tab-unit');
+
+  if (container.length === 0) {
+    logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
+    return [];
+  }
+
+  // Extract each article element
+  container.find('article').each((index, article) => {
+    const $article = $(article);
+
+    const unit = {
+      // Core identifiers
+      id: $article.attr('data-spaces-id'),
+      unit_code: $article.attr('data-spaces-unit'),
+      unit_id: $article.attr('data-spaces-unit-id'),
+
+      // Physical attributes
+      floor: $article.attr('data-spaces-unit-floor'),
+      area: $article.attr('data-spaces-sort-area'),
+      bed_count: $article.attr('data-spaces-sort-bed'),
+      bath_count: $article.attr('data-spaces-sort-bath'),
+
+      // Pricing
+      price: $article.attr('data-spaces-sort-price'),
+
+      // Availability
+      available: $article.attr('data-spaces-available'),
+      unavailable: $article.attr('data-spaces-unavailable'),
+      soonest: $article.attr('data-spaces-soonest'),
+      date_available: $article.attr('data-spaces-sort-date'),
+
+      // Plan information
+      plan_id: $article.attr('data-spaces-plan-id'),
+      plan_name: $article.attr('data-spaces-sort-plan-name'),
+
+      // Property information
+      obj_type: $article.attr('data-spaces-obj'),
+      community: $article.attr('data-spaces-community'),
+      asset: $article.attr('data-spaces-asset'),
+
+      // URLs
+      href: $article.attr('data-spaces-href'),
+      inventory_href: $article.attr('data-spaces-inventory-href'),
+
+      // Specials
+      specials_content: $article.attr('data-spaces-specials-content')
+    };
+
+    // Extract image URL from nested element if present
+    const imgElement = $article.find('img').first();
+    if (imgElement.length > 0) {
+      unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
+    }
+
+    units.push(unit);
+  });
+
+  logger.info('Units parsed', { count: units.length });
+
+  // Deduplicate by unit_code (in case of duplicate articles)
+  const seen = new Set();
+  const deduplicated = units.filter(unit => {
+    if (!unit.unit_code || seen.has(unit.unit_code)) {
+      return false;
+    }
+    seen.add(unit.unit_code);
+    return true;
+  });
+
+  if (deduplicated.length < units.length) {
+    logger.warn('Duplicate units removed', {
+      original: units.length,
+      deduplicated: deduplicated.length
+    });
+  }
+
+  return deduplicated;
+}
+
+// ============================================================
+// Data Type Conversion Helpers
+// ============================================================
+
+/**
+ * Parse string to integer, return null for invalid
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed integer or null
+ */
+function parseInteger(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Parse string to positive integer, return null for 0 or invalid.
+ * Used for fields like area where 0 means "not available".
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed positive integer or null
+ */
+function parsePositiveInteger(value) {
+  const parsed = parseInteger(value);
+  return parsed === 0 ? null : parsed;
+}
+
+/**
+ * Parse value that could be integer or string identifier.
+ * Returns integer if cleanly parseable, otherwise trimmed string.
+ * @param {*} value - Value to parse
+ * @returns {number|string|null} Parsed integer, trimmed string, or null
+ */
+function parseIntegerOrString(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  // If it parses cleanly to an integer, return integer
+  if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
+    return parsed;
+  }
+  // Otherwise return as trimmed string
+  return String(value).trim();
+}
+
+/**
+ * Parse price, stripping non-numeric characters.
+ * Handles "$1,234" format and "Call for pricing" text.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed price or null
+ */
+function parsePrice(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+
+  // Check for "Call for pricing" or similar text
+  if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
+    return null;
+  }
+
+  // Strip non-numeric characters except decimal point
+  const cleaned = String(value).replace(/[^0-9.]/g, '');
+  const parsed = parseInt(cleaned, 10);
+
+  if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
+    return null;
+  }
+
+  // Negative prices are invalid
+  if (parsed < 0) {
+    return null;
+  }
+
+  return parsed;
+}
+
+/**
+ * Parse boolean from string.
+ * Handles "true"/"false"/"1"/"0" and actual boolean values.
+ * @param {*} value - Value to parse
+ * @returns {boolean|null} Parsed boolean or null
+ */
+function parseBoolean(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  const str = String(value).toLowerCase().trim();
+  if (str === 'true' || str === '1') return true;
+  if (str === 'false' || str === '0') return false;
+  return null;
+}
+
+/**
+ * Parse float value.
+ * Named parseFloatValue to avoid shadowing the global parseFloat.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed float or null
+ */
+function parseFloatValue(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Trim string, return null for empty.
+ * Converts non-string values to string before trimming.
+ * @param {*} value - Value to trim
+ * @returns {string|null} Trimmed string or null
+ */
+function trimString(value) {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  const trimmed = String(value).trim();
+  return trimmed === '' ? null : trimmed;
+}
+
+// ============================================================
+// Main Data Type Conversion Function
+// ============================================================
+
+/**
+ * Convert unit data types from strings to proper types.
+ * Applies the appropriate parser to each field based on its expected type.
+ * @param {Object} unit - Raw unit object with string values
+ * @returns {Object} Unit object with converted types
+ */
+function convertDataTypes(unit) {
+  return {
+    // Integer fields
+    id: parseInteger(unit.id),
+    unit_id: parseIntegerOrString(unit.unit_id),
+    floor: parseInteger(unit.floor),
+    area: parsePositiveInteger(unit.area),
+    bed_count: parseInteger(unit.bed_count),
+    plan_id: parseInteger(unit.plan_id),
+    asset: parseInteger(unit.asset),
+    date_available: parseInteger(unit.date_available),
+
+    // Float fields
+    bath_count: parseFloatValue(unit.bath_count),
+
+    // Price field (special handling)
+    price: parsePrice(unit.price),
+
+    // Boolean fields
+    available: parseBoolean(unit.available),
+    unavailable: parseBoolean(unit.unavailable),
+
+    // String fields (trim and preserve)
+    unit_code: trimString(unit.unit_code),
+    plan_name: trimString(unit.plan_name),
+    soonest: trimString(unit.soonest),
+    obj_type: trimString(unit.obj_type),
+    community: trimString(unit.community),
+    href: trimString(unit.href),
+    inventory_href: trimString(unit.inventory_href),
+    image_url: trimString(unit.image_url),
+    specials_content: trimString(unit.specials_content)
+  };
+}
+
+// ============================================================
+// Date Helpers
+// ============================================================
+
+/**
+ * Calculate yesterday's date from a given date string.
+ * @param {string} dateStr - Date in YYYY-MM-DD format
+ * @returns {string} Yesterday's date in YYYY-MM-DD format
+ */
+function getYesterday(dateStr) {
+  const date = new Date(dateStr + 'T00:00:00Z');
+  date.setUTCDate(date.getUTCDate() - 1);
+  return date.toISOString().split('T')[0];
+}
+
+// ============================================================
+// Database Operations
+// ============================================================
+
+/**
+ * Upsert unit records to database using bulkWrite.
+ * Each unit is matched by unit_code as the unique key.
+ * Sets last_scraped and data_source on every update.
+ * Sets first_seen only on initial insert via $setOnInsert.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Bulk write result
+ */
+async function upsertUnits(db, units, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+  const now = new Date().toISOString();
+
+  const operations = units.map(unit => ({
+    updateOne: {
+      filter: { unit_code: unit.unit_code },
+      update: {
+        $set: {
+          ...unit,
+          last_scraped: now,
+          data_source: 'web_scraper'
+        },
+        $setOnInsert: {
+          first_seen: now
+        }
+      },
+      upsert: true
+    }
+  }));
+
+  if (operations.length === 0) {
+    logger.warn('No units to upsert');
+    return { modifiedCount: 0, upsertedCount: 0 };
+  }
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Units upserted', {
+      matched: result.matchedCount,
+      modified: result.modifiedCount,
+      upserted: result.upsertedCount
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to upsert units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Insert price records for today using bulkWrite with upsert.
+ * Uses unit_code + date_checked as the composite key to prevent
+ * duplicate price records for the same unit on the same day.
+ * Re-running on the same day updates existing records (idempotent).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {string} date - Date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Insert result with insertedCount
+ */
+async function insertPrices(db, units, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+  const now = new Date().toISOString();
+
+  // Filter out units without prices
+  const unitsWithPrices = units.filter(u => u.price !== null);
+
+  const priceRecords = unitsWithPrices.map(unit => ({
+    unit_code: unit.unit_code,
+    date_checked: date,
+    price: unit.price,
+    last_updated: now,
+    data_source: 'web_scraper'
+  }));
+
+  if (priceRecords.length === 0) {
+    logger.warn('No price records to insert');
+    return { insertedCount: 0 };
+  }
+
+  // Use updateOne with upsert to handle re-runs on same day
+  const operations = priceRecords.map(record => ({
+    updateOne: {
+      filter: {
+        unit_code: record.unit_code,
+        date_checked: record.date_checked
+      },
+      update: { $set: record },
+      upsert: true
+    }
+  }));
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Prices inserted', {
+      inserted: result.upsertedCount,
+      updated: result.modifiedCount
+    });
+
+    return { insertedCount: result.upsertedCount + result.modifiedCount };
+
+  } catch (error) {
+    logger.error('Failed to insert prices', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Mark units not in current scrape as stale/unavailable.
+ * Uses updateMany to set available: false and marked_stale_date
+ * for all units whose unit_code is NOT in the current scrape
+ * and that are currently available.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Set<string>} currentUnitCodes - Unit codes from current scrape
+ * @param {string} date - Current date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Update result
+ */
+async function markStaleUnits(db, currentUnitCodes, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+
+  try {
+    const result = await collection.updateMany(
+      {
+        unit_code: { $nin: [...currentUnitCodes] },
+        available: true
+      },
+      {
+        $set: {
+          available: false,
+          marked_stale_date: date
+        }
+      }
+    );
+
+    logger.info('Stale units marked', { count: result.modifiedCount });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to mark stale units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Update daily summary document.
+ * Upserts by date field (YYYY-MM-DD format).
+ * Fetches yesterday's summary for comparison metrics.
+ * Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} summaryData - Summary data
+ * @param {string} summaryData.date - Date in YYYY-MM-DD format
+ * @param {Array<string>} summaryData.newUnits - Unit codes added today
+ * @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
+ * @param {number} summaryData.staleUnitsCount - Count of stale units
+ * @param {number} summaryData.totalAvailable - Total available units today
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} MongoDB updateOne result
+ */
+async function updateDailySummary(db, summaryData, logger) {
+  const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
+  const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
+
+  // Calculate yesterday's date for comparison
+  const yesterday = getYesterday(date);
+
+  // Get yesterday's summary for comparison
+  const yesterdaySummary = await collection.findOne({ date: yesterday });
+  const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
+
+  const summary = {
+    date,
+    timestamp: new Date().toISOString(),
+    new_units: newUnits,
+    rented_units: rentedUnits,
+    stale_units: [], // Stale units list is not tracked per PRD
+    new_units_count: newUnits.length,
+    rented_units_count: rentedUnits.length,
+    stale_units_count: staleUnitsCount,
+    net_change: newUnits.length - rentedUnits.length,
+    total_available_today: totalAvailable,
+    total_available_yesterday: yesterdayTotal,
+    turnover_rate: yesterdayTotal > 0
+      ? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
+      : 0
+  };
+
+  try {
+    const result = await collection.updateOne(
+      { date },
+      { $set: summary },
+      { upsert: true }
+    );
+
+    logger.info('Daily summary updated', {
+      date,
+      newUnits: newUnits.length,
+      rentedUnits: rentedUnits.length,
+      totalAvailable
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to update daily summary', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+// ============================================================
+// Additional Helpers for runScrape Orchestration
+// ============================================================
+
+/**
+ * Get today's date in YYYY-MM-DD format (UTC).
+ * @returns {string} Today's date string
+ */
+function getTodayUTC() {
+  return new Date().toISOString().split('T')[0];
+}
+
+/**
+ * Get the set of unit codes that had price records yesterday.
+ * Used to calculate new and rented units by comparison.
+ * @param {Db} db - MongoDB database instance
+ * @param {string} today - Today's date in YYYY-MM-DD format
+ * @returns {Promise<Set<string>>} Set of unit codes from yesterday
+ */
+async function getYesterdayUnitCodes(db, today) {
+  const yesterday = getYesterday(today);
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+
+  const yesterdayRecords = await collection
+    .find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
+    .toArray();
+
+  return new Set(yesterdayRecords.map(r => r.unit_code));
+}
+
+// ============================================================
+// Error Sanitization
+// ============================================================
+
+/**
+ * Sanitize an error object to remove sensitive information before storage.
+ * Removes file paths, connection strings, and credential patterns while
+ * preserving the error type and a useful general description for debugging.
+ *
+ * @param {Error} error - Error object to sanitize
+ * @returns {Object} Sanitized error with name, message, and optionally stack
+ */
+function sanitizeError(error) {
+  const sanitized = {
+    name: error.name || 'Error',
+    message: sanitizeMessage(error.message || ''),
+  };
+
+  if (error.stack) {
+    sanitized.stack = sanitizeMessage(error.stack);
+  }
+
+  return sanitized;
+}
+
+/**
+ * Sanitize a string message by removing sensitive patterns.
+ * @param {string} message - Raw error message
+ * @returns {string} Sanitized message
+ */
+function sanitizeMessage(message) {
+  let result = message;
+
+  // Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
+  result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
+
+  // Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
+  result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
+
+  // Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
+  result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
+
+  // Remove Windows-style absolute paths (C:\..., D:\...)
+  result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
+
+  return result;
+}
+
+// ============================================================
+// Scraper Run History
+// ============================================================
+
+/**
+ * Record scraper run to history collection.
+ * Called in the finally block of runScrape() to persist run metadata.
+ * This function must NOT throw errors - history recording should never break the scraper.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object|null>} Insert result, or null on failure
+ */
+async function recordScraperRun(db, runData, logger) {
+  try {
+    const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
+    const result = await collection.insertOne({
+      ...runData,
+      recordedAt: new Date()
+    });
+
+    return result;
+
+  } catch (error) {
+    // Log but don't throw - recording history should not break scraper
+    logger.error('Failed to record scraper run', { errorMessage: error.message });
+    return null;
+  }
+}
+
+// ============================================================
+// Main Orchestration Function
+// ============================================================
+
+/**
+ * Execute a complete scrape operation.
+ * Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} options - Scrape options
+ * @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
+ * @param {string} [options.jobId] - Optional job ID (generated if not provided)
+ * @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
+ * @param {string} [options.htmlContent] - Use provided HTML instead of fetching
+ * @returns {Promise<Object>} Scrape result with status and metrics
+ */
+async function runScrape(db, options = {}) {
+  const jobId = options.jobId || crypto.randomUUID();
+  const trigger = options.trigger || 'manual';
+  const dryRun = options.dryRun || false;
+  const htmlContent = options.htmlContent || null;
+  const logger = createLogger(jobId);
+  const startTime = Date.now();
+
+  let result = {
+    jobId,
+    trigger,
+    dryRun,
+    status: 'running',
+    startedAt: new Date().toISOString(),
+    completedAt: null,
+    duration: null,
+    unitsProcessed: 0,
+    pricesInserted: 0,
+    newUnitsCount: 0,
+    rentedUnitsCount: 0,
+    staleUnitsCount: 0,
+    errors: []
+  };
+
+  try {
+    logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
+
+    // Step 1: Fetch HTML (or use provided content for testing)
+    const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
+
+    // Step 2: Parse units
+    const rawUnits = parseUnits(html, logger);
+
+    if (rawUnits.length === 0) {
+      logger.warn('No units found in HTML - possible structure change');
+      result.errors.push('No units found in HTML');
+    }
+
+    // Step 3: Convert data types
+    const units = rawUnits.map(unit => convertDataTypes(unit));
+    result.unitsProcessed = units.length;
+
+    // Step 4: Database operations
+    const today = getTodayUTC();
+
+    // Get yesterday's unit codes for comparison
+    const yesterdayUnits = await getYesterdayUnitCodes(db, today);
+
+    // Determine new and rented units
+    const currentUnitCodes = new Set(units.map(u => u.unit_code));
+    const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
+    const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
+
+    result.newUnitsCount = newUnits.length;
+    result.rentedUnitsCount = rentedUnits.length;
+
+    // Database operations (skip if dryRun)
+    if (dryRun) {
+      logger.info('Dry run mode - skipping database writes', {
+        wouldUpsert: units.length,
+        wouldInsertPrices: units.filter(u => u.price !== null).length
+      });
+      result.pricesInserted = 0;
+      result.staleUnitsCount = 0;
+    } else {
+      // Upsert units
+      await upsertUnits(db, units, logger);
+
+      // Insert prices
+      const pricesResult = await insertPrices(db, units, today, logger);
+      result.pricesInserted = pricesResult.insertedCount;
+
+      // Mark stale units
+      const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
+      result.staleUnitsCount = staleResult.modifiedCount;
+
+      // Update daily summary
+      await updateDailySummary(db, {
+        date: today,
+        newUnits: newUnits.map(u => u.unit_code),
+        rentedUnits,
+        staleUnitsCount: result.staleUnitsCount,
+        totalAvailable: units.length
+      }, logger);
+    }
+
+    result.status = 'success';
+
+  } catch (error) {
+    const cleanError = sanitizeError(error);
+    logger.error('Scrape failed', {
+      errorType: cleanError.name,
+      errorMessage: cleanError.message
+    });
+    result.status = 'failed';
+    result.errors.push(cleanError.message);
+
+  } finally {
+    result.completedAt = new Date().toISOString();
+    result.duration = Date.now() - startTime;
+
+    // Record run to history (always runs, even on failure)
+    await recordScraperRun(db, result, logger);
+
+    logger.info('Scrape completed', {
+      status: result.status,
+      duration: result.duration,
+      unitsProcessed: result.unitsProcessed,
+      pricesInserted: result.pricesInserted
+    });
+  }
+
+  return result;
+}
+
+module.exports = {
+  runScrape,
+  fetchPage,
+  parseUnits,
+  convertDataTypes,
+  upsertUnits,
+  insertPrices,
+  markStaleUnits,
+  updateDailySummary,
+  recordScraperRun,
+  // Export helpers for testing
+  getTodayUTC,
+  getYesterdayUnitCodes,
+  getYesterday,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString,
+  isRetryableError,
+  sleep,
+  sanitizeError
+};

test-endpoints.js

@@ -0,0 +1,680 @@
+#!/usr/bin/env node
+
+/**
+ * Comprehensive API Endpoint Test Script
+ * Tests all apartment dashboard endpoints with enhanced redirect handling
+ * Usage: node test-endpoints.js
+ */
+
+const https = require('https');
+const http = require('http');
+
+const BASE_URL = 'apartments.maverickapplications.com';
+const API_PREFIX = '/api';
+
+// ANSI color codes for console output
+const colors = {
+  reset: '\x1b[0m',
+  bright: '\x1b[1m',
+  red: '\x1b[31m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  magenta: '\x1b[35m',
+  cyan: '\x1b[36m'
+};
+
+// Test results tracking
+let testResults = {
+  passed: 0,
+  failed: 0,
+  total: 0,
+  details: []
+};
+
+// Configuration for different protocols
+let currentConfig = {
+  protocol: 'http',
+  port: 80,
+  module: http
+};
+
+/**
+ * Make HTTP/HTTPS request with redirect handling
+ */
+function makeRequest(path, method = 'GET', maxRedirects = 5) {
+  return new Promise((resolve, reject) => {
+    function attemptRequest(currentPath, redirectCount = 0) {
+      const options = {
+        hostname: BASE_URL,
+        port: currentConfig.port,
+        path: currentPath,
+        method: method,
+        headers: {
+          'User-Agent': 'Apartment-API-Tester/1.0',
+          'Accept': 'application/json',
+          'Host': BASE_URL
+        },
+        timeout: 15000 // 15 second timeout
+      };
+
+      const req = currentConfig.module.request(options, (res) => {
+        let data = '';
+        
+        // Handle redirects
+        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          if (redirectCount >= maxRedirects) {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: null,
+              rawData: `Too many redirects (${redirectCount})`,
+              redirectLocation: res.headers.location
+            });
+            return;
+          }
+          
+          let redirectUrl = res.headers.location;
+          console.log(`${colors.yellow}    → Redirect ${res.statusCode} to: ${redirectUrl}${colors.reset}`);
+          
+          // Handle relative redirects
+          if (redirectUrl.startsWith('/')) {
+            attemptRequest(redirectUrl, redirectCount + 1);
+            return;
+          }
+          
+          // Handle absolute redirects (change protocol if needed)
+          if (redirectUrl.startsWith('https://') && currentConfig.protocol === 'http') {
+            console.log(`${colors.yellow}    → Switching to HTTPS due to redirect${colors.reset}`);
+            currentConfig = { protocol: 'https', port: 443, module: https };
+            const urlPath = redirectUrl.replace(`https://${BASE_URL}`, '');
+            attemptRequest(urlPath, redirectCount + 1);
+            return;
+          }
+          
+          if (redirectUrl.startsWith('http://') && currentConfig.protocol === 'https') {
+            console.log(`${colors.yellow}    → Switching to HTTP due to redirect${colors.reset}`);
+            currentConfig = { protocol: 'http', port: 80, module: http };
+            const urlPath = redirectUrl.replace(`http://${BASE_URL}`, '');
+            attemptRequest(urlPath, redirectCount + 1);
+            return;
+          }
+          
+          // For other absolute URLs, extract the path
+          try {
+            const url = new URL(redirectUrl);
+            if (url.hostname === BASE_URL) {
+              attemptRequest(url.pathname + url.search, redirectCount + 1);
+              return;
+            }
+          } catch (e) {
+            // If URL parsing fails, treat as relative
+            attemptRequest(redirectUrl, redirectCount + 1);
+            return;
+          }
+        }
+        
+        res.on('data', (chunk) => {
+          data += chunk;
+        });
+        
+        res.on('end', () => {
+          try {
+            const jsonData = data ? JSON.parse(data) : {};
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: jsonData,
+              rawData: data,
+              finalUrl: `${currentConfig.protocol}://${BASE_URL}${currentPath}`
+            });
+          } catch (e) {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: null,
+              rawData: data,
+              parseError: e.message,
+              finalUrl: `${currentConfig.protocol}://${BASE_URL}${currentPath}`
+            });
+          }
+        });
+      });
+
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error(`Request timeout after 15s for ${currentConfig.protocol}://${BASE_URL}${currentPath}`));
+      });
+
+      req.on('error', (error) => {
+        reject(new Error(`${error.message} (${currentConfig.protocol}://${BASE_URL}${currentPath})`));
+      });
+
+      req.end();
+    }
+    
+    attemptRequest(path);
+  });
+}
+
+/**
+ * Test connection and protocol detection
+ */
+async function detectBestProtocol() {
+  console.log(`${colors.magenta}🔍 Detecting best protocol and configuration...${colors.reset}\n`);
+  
+  const testConfigs = [
+    { protocol: 'http', port: 80, module: http, name: 'HTTP (port 80)' },
+    { protocol: 'https', port: 443, module: https, name: 'HTTPS (port 443)' },
+    { protocol: 'http', port: 8080, module: http, name: 'HTTP (port 8080)' },
+    { protocol: 'http', port: 3000, module: http, name: 'HTTP (port 3000)' }
+  ];
+  
+  for (const config of testConfigs) {
+    currentConfig = config;
+    console.log(`${colors.cyan}Testing ${config.name}...${colors.reset}`);
+    
+    try {
+      const response = await makeRequest('/health');
+      console.log(`${colors.green}✅ ${config.name} - Status: ${response.statusCode}${colors.reset}`);
+      
+      if (response.statusCode === 200) {
+        console.log(`${colors.green}🎯 Using ${config.name} for all tests${colors.reset}\n`);
+        return true;
+      } else if (response.statusCode >= 300 && response.statusCode < 400) {
+        console.log(`${colors.yellow}⚠️  ${config.name} redirects to: ${response.headers.location || 'unknown'}${colors.reset}`);
+        // Don't return, try other configs first
+      }
+    } catch (error) {
+      console.log(`${colors.red}❌ ${config.name} - ${error.message}${colors.reset}`);
+    }
+  }
+  
+  // If no 200 response found, use the first config that doesn't error
+  console.log(`${colors.yellow}⚠️  No perfect match found, using HTTP as fallback${colors.reset}\n`);
+  currentConfig = testConfigs[0];
+  return false;
+}
+
+/**
+ * Test a single endpoint with enhanced debugging
+ */
+async function testEndpoint(name, path, expectedFields = [], validStatusCodes = [200]) {
+  testResults.total++;
+  console.log(`${colors.cyan}Testing: ${colors.bright}${name}${colors.reset}`);
+  console.log(`${colors.blue}  Path: ${currentConfig.protocol}://${BASE_URL}${path}${colors.reset}`);
+  
+  try {
+    const response = await makeRequest(path);
+    const { statusCode, data, rawData, parseError, finalUrl, redirectLocation } = response;
+    
+    // Show final URL if different from initial
+    if (finalUrl && finalUrl !== `${currentConfig.protocol}://${BASE_URL}${path}`) {
+      console.log(`${colors.yellow}  Final URL: ${finalUrl}${colors.reset}`);
+    }
+    
+    // Show redirect info for debugging
+    if (statusCode >= 300 && statusCode < 400) {
+      console.log(`${colors.yellow}  Redirect Status: ${statusCode}${colors.reset}`);
+      if (redirectLocation) {
+        console.log(`${colors.yellow}  Redirect Location: ${redirectLocation}${colors.reset}`);
+      }
+    }
+    
+    // Check status code
+    if (!validStatusCodes.includes(statusCode)) {
+      // For 301/302 redirects, provide helpful info
+      if (statusCode === 301 || statusCode === 302) {
+        throw new Error(`Redirect ${statusCode} - Server redirecting to: ${redirectLocation || 'unknown'}. This might indicate wrong protocol or path format.`);
+      }
+      throw new Error(`Expected status ${validStatusCodes.join(' or ')}, got ${statusCode}`);
+    }
+    
+    // Check if response is valid JSON
+    if (parseError) {
+      console.log(`${colors.yellow}  Raw response: ${rawData.substring(0, 200)}...${colors.reset}`);
+      throw new Error(`JSON parse error: ${parseError}`);
+    }
+    
+    // Check for expected fields if data is an object
+    if (expectedFields.length > 0 && data && typeof data === 'object') {
+      const missingFields = [];
+      
+      if (Array.isArray(data)) {
+        // If it's an array, check the first item
+        if (data.length > 0) {
+          expectedFields.forEach(field => {
+            if (!(field in data[0])) {
+              missingFields.push(field);
+            }
+          });
+        }
+      } else {
+        // If it's an object, check directly
+        expectedFields.forEach(field => {
+          if (!(field in data)) {
+            missingFields.push(field);
+          }
+        });
+      }
+      
+      if (missingFields.length > 0) {
+        console.log(`${colors.yellow}  ⚠️  Missing expected fields: ${missingFields.join(', ')}${colors.reset}`);
+      }
+    }
+    
+    // Success
+    testResults.passed++;
+    console.log(`${colors.green}  ✅ PASS${colors.reset}`);
+    console.log(`${colors.green}     Status: ${statusCode}${colors.reset}`);
+    console.log(`${colors.green}     Protocol: ${currentConfig.protocol.toUpperCase()}${colors.reset}`);
+    
+    if (Array.isArray(data)) {
+      console.log(`${colors.green}     Records: ${data.length}${colors.reset}`);
+    } else if (data && typeof data === 'object') {
+      console.log(`${colors.green}     Keys: ${Object.keys(data).length}${colors.reset}`);
+    }
+    
+    testResults.details.push({
+      name,
+      path,
+      status: 'PASS',
+      statusCode,
+      protocol: currentConfig.protocol,
+      dataType: Array.isArray(data) ? 'array' : typeof data,
+      recordCount: Array.isArray(data) ? data.length : null
+    });
+    
+  } catch (error) {
+    testResults.failed++;
+    console.log(`${colors.red}  ❌ FAIL${colors.reset}`);
+    console.log(`${colors.red}     Error: ${error.message}${colors.reset}`);
+    console.log(`${colors.red}     Protocol: ${currentConfig.protocol.toUpperCase()}${colors.reset}`);
+    
+    testResults.details.push({
+      name,
+      path,
+      status: 'FAIL',
+      protocol: currentConfig.protocol,
+      error: error.message
+    });
+  }
+  
+  console.log(''); // Empty line for readability
+}
+
+/**
+ * Test with curl-equivalent settings
+ */
+async function testWithCurlEquivalent() {
+  console.log(`${colors.yellow}🔄 Testing with curl-equivalent settings...${colors.reset}\n`);
+  
+  // Test the exact same way curl would
+  const curlTests = [
+    { url: 'http://apartments.maverickapplications.com/health', description: 'Direct HTTP health check' },
+    { url: 'https://apartments.maverickapplications.com/health', description: 'Direct HTTPS health check' },
+    { url: 'http://apartments.maverickapplications.com/api/daily-summary', description: 'HTTP API endpoint' },
+    { url: 'https://apartments.maverickapplications.com/api/daily-summary', description: 'HTTPS API endpoint' }
+  ];
+  
+  for (const test of curlTests) {
+    console.log(`${colors.cyan}Testing: ${test.description}${colors.reset}`);
+    console.log(`${colors.blue}URL: ${test.url}${colors.reset}`);
+    
+    try {
+      const isHttps = test.url.startsWith('https');
+      const url = new URL(test.url);
+      
+      currentConfig = {
+        protocol: isHttps ? 'https' : 'http',
+        port: isHttps ? 443 : 80,
+        module: isHttps ? https : http
+      };
+      
+      const response = await makeRequest(url.pathname + url.search);
+      
+      console.log(`${colors.green}✅ Success - Status: ${response.statusCode}${colors.reset}`);
+      if (response.statusCode === 200 && response.data) {
+        console.log(`${colors.green}   Data type: ${Array.isArray(response.data) ? 'array' : typeof response.data}${colors.reset}`);
+        if (Array.isArray(response.data)) {
+          console.log(`${colors.green}   Records: ${response.data.length}${colors.reset}`);
+        } else if (typeof response.data === 'object') {
+          console.log(`${colors.green}   Keys: ${Object.keys(response.data).join(', ')}${colors.reset}`);
+        }
+      }
+      
+      // If we found a working endpoint, use this config for remaining tests
+      if (response.statusCode === 200) {
+        console.log(`${colors.green}🎯 Found working configuration: ${currentConfig.protocol.toUpperCase()}${colors.reset}\n`);
+        return true;
+      }
+      
+    } catch (error) {
+      console.log(`${colors.red}❌ Failed: ${error.message}${colors.reset}`);
+    }
+    console.log('');
+  }
+  
+  return false;
+}
+
+/**
+ * Test all endpoints
+ */
+async function runAllTests() {
+  console.log(`${colors.magenta}${colors.bright}🧪 APARTMENT API ENDPOINT TESTS${colors.reset}`);
+  console.log(`${colors.magenta}Base URL: ${BASE_URL}${colors.reset}`);
+  console.log(`${colors.magenta}Testing ${new Date().toISOString()}${colors.reset}\n`);
+  
+  // First, detect the best protocol
+  await detectBestProtocol();
+  
+  // Test a few different path formats to see which works
+  console.log(`${colors.magenta}🔍 Testing different path formats...${colors.reset}\n`);
+  
+  const pathVariations = [
+    '/health',
+    '/api/health', 
+    '/',
+    '/api/'
+  ];
+  
+  for (const testPath of pathVariations) {
+    try {
+      console.log(`${colors.cyan}Testing path: ${testPath}${colors.reset}`);
+      const response = await makeRequest(testPath);
+      console.log(`${colors.green}  Status: ${response.statusCode}${colors.reset}`);
+      if (response.statusCode === 200 && response.data) {
+        console.log(`${colors.green}  Response preview: ${JSON.stringify(response.data).substring(0, 100)}...${colors.reset}`);
+      }
+      if (response.rawData && response.statusCode !== 200) {
+        console.log(`${colors.yellow}  Raw response: ${response.rawData.substring(0, 200)}...${colors.reset}`);
+      }
+    } catch (error) {
+      console.log(`${colors.red}  Error: ${error.message}${colors.reset}`);
+    }
+    console.log('');
+  }
+  
+  console.log(`${colors.magenta}🚀 Starting full endpoint tests...${colors.reset}\n`);
+  
+  // Health check (try both with and without /api prefix)
+  await testEndpoint(
+    'Health Check',
+    '/health',
+    ['status', 'timestamp'],
+    [200, 404]
+  );
+  
+  // Try health check with /api prefix if first one failed
+  if (testResults.details[testResults.details.length - 1].status === 'FAIL') {
+    await testEndpoint(
+      'Health Check (with /api)',
+      '/api/health',
+      ['status', 'timestamp'],
+      [200, 404]
+    );
+  }
+  
+  // Original endpoints
+  await testEndpoint(
+    'Daily Summary',
+    `${API_PREFIX}/daily-summary`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Price History',
+    `${API_PREFIX}/price-history`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Available Units',
+    `${API_PREFIX}/available-units`,
+    ['unitCode'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Recent Activity',
+    `${API_PREFIX}/recent-activity`,
+    ['type'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Plan Statistics',
+    `${API_PREFIX}/plan-stats`,
+    ['name'],
+    [200, 404]
+  );
+  
+  // Enhanced endpoints (these might not exist yet)
+  await testEndpoint(
+    'Best Deals',
+    `${API_PREFIX}/best-deals`,
+    ['unitCode', 'planName', 'currentPrice'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Price Drops',
+    `${API_PREFIX}/price-drops`,
+    ['unitCode', 'currentPrice'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Stale Inventory',
+    `${API_PREFIX}/stale-inventory`,
+    ['unitCode', 'daysAvailable'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Market Insights',
+    `${API_PREFIX}/market-insights`,
+    ['avgDaysOnMarket'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Enhanced Available Units',
+    `${API_PREFIX}/available-units-enhanced`,
+    ['unitCode', 'amenities'],
+    [200, 404, 500]
+  );
+  
+  // Test with query parameters
+  await testEndpoint(
+    'Price History (7 days)',
+    `${API_PREFIX}/price-history?days=7`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Best Deals (limit 3)',
+    `${API_PREFIX}/best-deals?limit=3`,
+    [],
+    [200, 404, 500]
+  );
+  
+  // Test individual unit endpoint
+  await testEndpoint(
+    'Individual Unit Details',
+    `${API_PREFIX}/unit/A101`,
+    [],
+    [200, 404, 500]
+  );
+  
+  // Test edge cases
+  await testEndpoint(
+    'Invalid Endpoint',
+    `${API_PREFIX}/nonexistent`,
+    [],
+    [404, 405] // Should return 404 or 405
+  );
+  
+  // Print final results
+  printTestSummary();
+}
+
+/**
+ * Print test summary
+ */
+function printTestSummary() {
+  console.log(`${colors.magenta}${colors.bright}📊 TEST SUMMARY${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  console.log(`${colors.green}✅ Passed: ${testResults.passed}${colors.reset}`);
+  console.log(`${colors.red}❌ Failed: ${testResults.failed}${colors.reset}`);
+  console.log(`${colors.blue}📋 Total:  ${testResults.total}${colors.reset}`);
+  console.log(`${colors.yellow}📈 Success Rate: ${((testResults.passed / testResults.total) * 100).toFixed(1)}%${colors.reset}`);
+  console.log('');
+  
+  // Detailed results
+  console.log(`${colors.magenta}${colors.bright}📋 DETAILED RESULTS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  
+  testResults.details.forEach((test, index) => {
+    const status = test.status === 'PASS' ? 
+      `${colors.green}✅ PASS${colors.reset}` : 
+      `${colors.red}❌ FAIL${colors.reset}`;
+    
+    console.log(`${index + 1}. ${test.name}`);
+    console.log(`   ${status} - ${test.path}`);
+    
+    if (test.status === 'PASS') {
+      if (test.recordCount !== null) {
+        console.log(`   📊 ${test.recordCount} records returned`);
+      }
+    } else {
+      console.log(`   💥 ${test.error}`);
+    }
+    console.log('');
+  });
+  
+  // Recommendations
+  console.log(`${colors.magenta}${colors.bright}💡 RECOMMENDATIONS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  
+  const failedTests = testResults.details.filter(t => t.status === 'FAIL');
+  const enhancedEndpoints = failedTests.filter(t => 
+    t.path.includes('best-deals') || 
+    t.path.includes('price-drops') || 
+    t.path.includes('stale-inventory') || 
+    t.path.includes('market-insights') || 
+    t.path.includes('available-units-enhanced')
+  );
+  
+  if (enhancedEndpoints.length > 0) {
+    console.log(`${colors.yellow}⚠️  Enhanced endpoints not implemented yet:${colors.reset}`);
+    enhancedEndpoints.forEach(test => {
+      console.log(`   - ${test.path}`);
+    });
+    console.log(`${colors.blue}   💡 Add the missing routes from the enhanced server script${colors.reset}`);
+    console.log('');
+  }
+  
+  if (testResults.failed > enhancedEndpoints.length) {
+    console.log(`${colors.red}🚨 Core endpoints failing - check server configuration${colors.reset}`);
+    console.log('');
+  }
+  
+  if (testResults.passed === testResults.total) {
+    console.log(`${colors.green}🎉 All tests passed! API is fully functional.${colors.reset}`);
+  }
+}
+
+/**
+ * Debug curl command generator
+ */
+function generateCurlCommands() {
+  console.log(`${colors.magenta}${colors.bright}🔧 EQUIVALENT CURL COMMANDS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  console.log(`${colors.yellow}Try these curl commands to debug:${colors.reset}\n`);
+  
+  const endpoints = [
+    '/health',
+    '/api/health', 
+    '/api/daily-summary',
+    '/api/price-history',
+    '/api/available-units'
+  ];
+  
+  endpoints.forEach(endpoint => {
+    console.log(`${colors.cyan}# Test ${endpoint}${colors.reset}`);
+    console.log(`curl -v -H "Accept: application/json" http://${BASE_URL}${endpoint}`);
+    console.log(`curl -v -H "Accept: application/json" https://${BASE_URL}${endpoint}`);
+    console.log('');
+  });
+  
+  console.log(`${colors.yellow}Look for:${colors.reset}`);
+  console.log(`${colors.blue}- HTTP status codes (200 = success, 301/302 = redirect, 404 = not found)${colors.reset}`);
+  console.log(`${colors.blue}- Location headers in redirects${colors.reset}`);
+  console.log(`${colors.blue}- Response content-type${colors.reset}`);
+  console.log(`${colors.blue}- Server response headers${colors.reset}\n`);
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  try {
+    await runAllTests();
+    
+    // If all tests failed with redirects, try curl-equivalent testing
+    const allFailed = testResults.passed === 0 && testResults.failed > 0;
+    const hasRedirectErrors = testResults.details.some(test => 
+      test.error && test.error.includes('Redirect')
+    );
+    
+    if (allFailed && hasRedirectErrors) {
+      console.log(`${colors.yellow}🔍 All tests failed with redirects. Trying curl-equivalent approach...${colors.reset}\n`);
+      await testWithCurlEquivalent();
+    }
+    
+    // Generate curl commands for manual testing
+    generateCurlCommands();
+    
+  } catch (error) {
+    console.error(`${colors.red}💥 Fatal error: ${error.message}${colors.reset}`);
+    process.exit(1);
+  }
+  
+  // Exit with error code if tests failed
+  process.exit(testResults.failed > 0 ? 1 : 0);
+}
+
+// Handle graceful shutdown
+process.on('SIGINT', () => {
+  console.log(`\n${colors.yellow}🛑 Test interrupted by user${colors.reset}`);
+  process.exit(1);
+});
+
+// Add CLI argument parsing
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`
+${colors.bright}Apartment API Endpoint Tester${colors.reset}
+
+${colors.cyan}Usage:${colors.reset}
+  node test-endpoints.js [options]
+
+${colors.cyan}Options:${colors.reset}
+  --help, -h     Show this help message
+  --https        Force HTTPS testing
+  --verbose      Show detailed response data
+
+${colors.cyan}Examples:${colors.reset}
+  node test-endpoints.js
+  node test-endpoints.js --https
+  node test-endpoints.js --verbose
+`);
+  process.exit(0);
+}
+
+// Run the main function
+main();