Add GET /api/admin/scraper/history endpoint with pagination

Implement the scraper history endpoint that returns past scrape run
records from the scraper_runs collection. Features include:

- GET /api/admin/scraper/history protected by requireAuth + requireAdmin
- Pagination via limit (1-100, default 30) and offset (default 0) params
- Input validation returning 400 for invalid limit/offset values
- Results sorted by startedAt descending (newest first)
- Graceful error handling returning 503 on database failures
- 20 new tests covering auth, pagination, validation, and error cases

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,25 @@
+# Environment and secrets
+.env
+.env.*
+!.env.example
+
+# Dependencies
+node_modules
+
+# Git
+.git
+.gitignore
+
+# Development files
+*.log
+npm-debug.log*
+.DS_Store
+
+# Test files
+test-endpoints.js
+*.test.js
+__tests__
+
+# IDE
+.vscode
+.idea

.env.example

@@ -0,0 +1,20 @@
+# MongoDB Connection
+MONGO_URI=mongodb://username:password@host:27017
+
+# Google OAuth Credentials (from Google Cloud Console)
+GOOGLE_CLIENT_ID=your-client-id.apps.googleusercontent.com
+GOOGLE_CLIENT_SECRET=your-client-secret
+GOOGLE_CALLBACK_URL=https://apartments.maverickapplications.com/api/auth/google/callback
+
+# JWT Configuration
+# Generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
+JWT_SECRET=generate-a-secure-random-string-minimum-32-characters
+
+# Application URLs
+FRONTEND_URL=https://apartments.maverickapplications.com
+
+# Activity Logging Level: all, navigation, none
+ACTIVITY_LOG_LEVEL=all
+
+# Node Environment
+NODE_ENV=production

.github/workflows/deploy.yml

@@ -1,60 +1,368 @@
-name: Deploy Apartment API
+name: CI/CD Pipeline - Apartment API
 
 on:
   push:
     branches: [ main ]
+  pull_request:
   workflow_dispatch:
 
+env:
+  NODE_VERSION: '20'
+  IMAGE_NAME: apartment-api
+
 jobs:
-  deploy:
+  # ============================================================
+  # CI Job - Lint + Test in a single job (one checkout, one npm ci)
+  # ============================================================
+  ci:
+    name: Lint & Test
+    runs-on: ubuntu-latest
+
+    outputs:
+      lint_status: ${{ steps.lint.outcome }}
+      lint_output: ${{ steps.lint.outputs.lint_output }}
+      test_status: ${{ steps.test.outcome }}
+      test_output: ${{ steps.test.outputs.test_output }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run ESLint
+        id: lint
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm run lint 2>&1)
+          EXIT_CODE=$?
+          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+
+      - name: Run tests
+        id: test
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm test -- --runInBand 2>&1)
+          EXIT_CODE=$?
+          echo "test_output<<EOF" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+        env:
+          JWT_SECRET: test-jwt-secret-for-ci
+          NODE_ENV: test
+
+  # ============================================================
+  # Notify Job - Send results to n8n webhook
+  # ============================================================
+  notify:
+    name: Send Webhook Notification
+    runs-on: ubuntu-latest
+    needs: [ci]
+    if: always()
+
+    steps:
+      - name: Send results to n8n webhook
+        env:
+          LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
+          TEST_STATUS: ${{ needs.ci.outputs.test_status }}
+          RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
+          RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
+          GH_REPO: ${{ github.repository }}
+          GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+          GH_SHA: ${{ github.sha }}
+          GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
+          GH_ACTOR: ${{ github.actor }}
+          GH_EVENT: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
+        run: |
+          # Determine overall status
+          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
+            OVERALL_STATUS="success"
+          else
+            OVERALL_STATUS="failure"
+          fi
+
+          # Truncate outputs if too long (max 10000 chars each)
+          LINT_OUTPUT="${RAW_LINT_OUTPUT:0:10000}"
+          TEST_OUTPUT="${RAW_TEST_OUTPUT:0:10000}"
+
+          # Build JSON payload
+          PAYLOAD=$(jq -n \
+            --arg repo "$GH_REPO" \
+            --arg branch "$GH_BRANCH" \
+            --arg commit "$GH_SHA" \
+            --arg commit_short "${GH_SHA:0:7}" \
+            --arg commit_message "$GH_COMMIT_MSG" \
+            --arg author "$GH_ACTOR" \
+            --arg event "$GH_EVENT" \
+            --arg pr_number "$GH_PR_NUMBER" \
+            --arg run_id "$GH_RUN_ID" \
+            --arg run_url "$GH_RUN_URL" \
+            --arg overall_status "$OVERALL_STATUS" \
+            --arg lint_status "$LINT_STATUS" \
+            --arg lint_output "$LINT_OUTPUT" \
+            --arg test_status "$TEST_STATUS" \
+            --arg test_output "$TEST_OUTPUT" \
+            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            '{
+              repository: $repo,
+              branch: $branch,
+              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
+              commit: {
+                sha: $commit,
+                short_sha: $commit_short,
+                message: $commit_message,
+                author: $author
+              },
+              event: $event,
+              run: {
+                id: $run_id,
+                url: $run_url
+              },
+              status: $overall_status,
+              results: {
+                lint: {
+                  status: $lint_status,
+                  output: (if $lint_status != "success" then $lint_output else null end)
+                },
+                test: {
+                  status: $test_status,
+                  output: (if $test_status != "success" then $test_output else null end)
+                }
+              },
+              timestamp: $timestamp
+            }')
+
+          # Send webhook
+          curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "$WEBHOOK_URL" \
+            --fail --silent --show-error
+
+      - name: Fail if lint or tests failed
+        if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "   Lint: ${{ needs.ci.outputs.lint_status }}"
+          echo "   Test: ${{ needs.ci.outputs.test_status }}"
+          exit 1
+
+  # ============================================================
+  # Dependency Scan Job - Runs in parallel with tests
+  # ============================================================
+  scan-deps:
+    name: Scan Dependencies
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '18'
-        cache: 'npm'
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
 
-    - name: Install dependencies
-      run: npm ci
+      - name: Scan dependencies for vulnerabilities
+        run: |
+          trivy fs \
+            --exit-code 1 \
+            --ignore-unfixed \
+            --severity CRITICAL,HIGH \
+            --scanners vuln \
+            .
 
-    - name: Build Docker image
-      run: |
-        docker build -t apartment-api:${{ github.sha }} .
-        docker tag apartment-api:${{ github.sha }} apartment-api:latest
+  # ============================================================
+  # Build & Push Job - Build image and push to Harbor
+  # ============================================================
+  build:
+    name: Build & Push Image
+    runs-on: ubuntu-latest
+    needs: [ci, scan-deps, notify]
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
 
-    - name: Deploy to server
-      run: |
-        # Copy files to deployment directory
-        mkdir -p /media/stephen/Storage_Linux/infrastructure/services/apartment-api
-        cp -r . /media/stephen/Storage_Linux/infrastructure/services/apartment-api/
+    outputs:
+      image_tag: ${{ steps.set-tag.outputs.tag }}
+      full_image: ${{ steps.set-tag.outputs.full_image }}
 
-        # Navigate to deployment directory
-        cd /media/stephen/Storage_Linux/infrastructure/services/apartment-api
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-        # Stop existing container if running
-        docker compose down || true
+      - name: Set image tag
+        id: set-tag
+        run: |
+          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+          echo "tag=${SHORT_SHA}" >> $GITHUB_OUTPUT
+          echo "full_image=${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${SHORT_SHA}" >> $GITHUB_OUTPUT
 
-        # Start new container
-        docker compose up -d
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-        # Clean up old images
-        docker image prune -f
+      - name: Log in to Harbor
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.HARBOR_REGISTRY }}
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
 
-    - name: Verify deployment
-      run: |
-        # Wait for container to start
-        sleep 15
+      - name: Build and push Docker image
+        id: build-push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}:latest
 
-        # Check if container is running
-        docker ps | grep apartment-api
+      # ============================================================
+      # Vulnerability Scanning with Trivy
+      # ============================================================
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.58.0
 
-        # Test health endpoint
-        curl -f http://localhost:3000/health || echo "API may still be starting..."
+      - name: Run Trivy vulnerability scanner
+        run: |
+          trivy image \
+            --exit-code 1 \
+            --ignore-unfixed \
+            --vuln-type os,library \
+            --severity CRITICAL,HIGH \
+            --format table \
+            ${{ steps.set-tag.outputs.full_image }}
 
-        # Test API endpoint
-        curl -f http://localhost:3000/api/daily-summary || echo "API may still be loading data..."
+      - name: Run Trivy and output SARIF
+        if: always()
+        run: |
+          trivy image \
+            --format sarif \
+            --output trivy-results.sarif \
+            ${{ steps.set-tag.outputs.full_image }} || true
+
+      # ============================================================
+      # SBOM Generation with Syft
+      # ============================================================
+      - name: Generate SBOM with Syft
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.set-tag.outputs.full_image }}
+          format: spdx-json
+          output-file: sbom.spdx.json
+
+      # ============================================================
+      # Image Signing with Cosign
+      # ============================================================
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with Cosign
+        env:
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        run: |
+          cosign sign --key env://COSIGN_PRIVATE_KEY \
+            --yes \
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
+
+      # ============================================================
+      # Attach SBOM to image in Harbor
+      # ============================================================
+      - name: Attach SBOM to image
+        env:
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        run: |
+          cosign attest --key env://COSIGN_PRIVATE_KEY \
+            --type spdxjson \
+            --predicate sbom.spdx.json \
+            --yes \
+            ${{ secrets.HARBOR_REGISTRY }}/${{ secrets.HARBOR_PROJECT }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}
+
+  # ============================================================
+  # Deploy Job - Pull image and restart on production server
+  # ============================================================
+  deploy:
+    name: Deploy to Production
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+
+    steps:
+      - name: Deploy via SSH
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          port: ${{ secrets.SSH_PORT || 22 }}
+          envs: IMAGE
+          script_stop: true
+          script: |
+            set -e
+
+            # Image to deploy (using specific SHA tag, not latest)
+            IMAGE="${{ needs.build.outputs.full_image }}"
+            echo "Deploying image: $IMAGE"
+
+            # Navigate to deployment directory
+            cd ${{ secrets.DEPLOY_PATH }}
+
+            # Pull the specific image (using pre-configured Docker credentials)
+            docker pull "$IMAGE"
+
+            # Update and restart with new image
+            export IMAGE
+            docker compose up -d
+
+            # Clean up old images
+            docker image prune -f
+
+            # Wait for container to be healthy
+            echo "Waiting for container to start..."
+            sleep 10
+
+            # Verify container is running
+            if docker compose ps | grep -q "Up"; then
+              echo "Container is running successfully"
+            else
+              echo "ERROR: Container failed to start"
+              docker compose logs --tail=50
+              exit 1
+            fi
+
+      - name: Verify deployment
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          port: ${{ secrets.SSH_PORT || 22 }}
+          script: |
+            # Test health endpoint
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://apartments.maverickapplications.com/api/health || echo "000")
+
+            if [ "$HTTP_CODE" = "200" ]; then
+              echo "Health check passed (HTTP $HTTP_CODE)"
+            else
+              echo "WARNING: Health check returned HTTP $HTTP_CODE"
+              echo "Container may still be initializing..."
+              exit 1
+            fi

AUTH.md

@@ -0,0 +1,717 @@
+# Google OAuth Authentication Implementation
+
+## Overview
+
+This document details the Google OAuth authentication implementation for the Apartment Dashboard application. **All pages and API endpoints require authentication** - unauthenticated users are redirected to a login page.
+
+> **Status:** Core authentication is complete and deployed. Admin dashboard and future enhancements are documented but not yet implemented.
+
+---
+
+## Table of Contents
+
+0. [Project Context](#0-project-context)
+1. [Implementation Status](#1-implementation-status)
+2. [Authentication Scope](#2-authentication-scope)
+3. [Technical Architecture](#3-technical-architecture)
+4. [Implementation Details](#4-implementation-details)
+5. [User Activity Logging](#5-user-activity-logging)
+6. [Security Measures](#6-security-measures)
+7. [Deployment Configuration](#7-deployment-configuration)
+8. [Test Checklist](#8-test-checklist)
+9. [Future: Admin Dashboard](#9-future-admin-dashboard)
+10. [Future: Adding Apple Sign-In](#10-future-adding-apple-sign-in)
+11. [Future: Partial Public Access](#11-future-partial-public-access)
+
+---
+
+## 0. Project Context
+
+### Project Structure
+
+```
+TowersPriceWebApp/
+├── apartment-dashboard/              # Frontend (React/Vite)
+│   ├── src/
+│   │   ├── App.jsx                   # Main app (~1300 lines)
+│   │   ├── main.jsx                  # Entry point (wrapped with AuthProvider)
+│   │   ├── index.css                 # Tailwind imports
+│   │   ├── context/
+│   │   │   └── AuthContext.jsx       # Auth state management
+│   │   ├── hooks/
+│   │   │   ├── useAuth.js            # Auth hook
+│   │   │   └── useActivityTracker.js # Activity tracking hook
+│   │   ├── pages/
+│   │   │   └── Login.jsx             # Login page with Google button
+│   │   └── components/
+│   │       └── ProtectedRoute.jsx    # Route guard component
+│   ├── vite.config.js
+│   ├── package.json
+│   ├── Dockerfile
+│   └── nginx.conf
+│
+├── apartment-dashboard-api/          # Backend (Express.js)
+│   ├── server.js                     # Main server (~1700 lines)
+│   ├── config/
+│   │   └── auth.js                   # JWT, cookie, OAuth config
+│   ├── models/
+│   │   └── user.js                   # User CRUD operations
+│   ├── middleware/
+│   │   ├── passport.js               # Google OAuth strategy
+│   │   └── auth.js                   # JWT verification middleware
+│   ├── routes/
+│   │   ├── auth.js                   # OAuth routes
+│   │   └── activity.js               # Activity logging routes
+│   ├── services/
+│   │   └── activityLogger.js         # Activity logging service
+│   ├── .env.example                  # Environment template
+│   ├── .dockerignore                 # Prevents secrets in image
+│   ├── package.json
+│   ├── Dockerfile
+│   └── docker-compose.yml
+│
+└── AUTH.md                           # This file
+```
+
+### Tech Stack
+
+| Layer | Technology | Version |
+|-------|------------|---------|
+| Frontend Framework | React | 19.1.0 |
+| Frontend Build | Vite | 7.0.4 |
+| Frontend Routing | React Router DOM | 7.12.0 |
+| Frontend Styling | Tailwind CSS | 3.4.17 |
+| Frontend Charts | Recharts | 3.1.0 |
+| Backend Framework | Express.js | 4.18.2 |
+| Database | MongoDB | 6.3.0 (driver) |
+| Auth Library | Passport.js | passport-google-oauth20 |
+| Token Management | jsonwebtoken | HTTP-only cookies |
+| Reverse Proxy | Traefik | (with Let's Encrypt SSL) |
+| Static Server | Nginx | (serves built React app) |
+
+### URLs and Domains
+
+| Environment | Frontend URL | API URL |
+|-------------|--------------|---------|
+| Production | `https://apartments.maverickapplications.com` | `https://apartments.maverickapplications.com/api` |
+| Development | `http://localhost:5173` (Vite) | `http://localhost:3000` |
+
+### MongoDB Details
+
+| Setting | Value |
+|---------|-------|
+| Database name | `apartments` |
+| Data collections | `units_migration_test`, `unit_prices_migration_test`, `daily_summaries` |
+| Auth collections | `users`, `user_activity` |
+
+---
+
+## 1. Implementation Status
+
+### Completed
+
+| Feature | Status | Notes |
+|---------|--------|-------|
+| Google OAuth login | Done | Passport.js with state parameter CSRF protection |
+| JWT in HTTP-only cookies | Done | 7-day expiry with sliding window refresh |
+| User model with upsert | Done | findOneAndUpdate with $setOnInsert pattern |
+| Protected API endpoints | Done | All data endpoints require valid JWT |
+| Activity logging | Done | Configurable levels, TTL indexes for cleanup |
+| Frontend auth context | Done | React Context with useAuth hook |
+| Protected routes | Done | ProtectedRoute component with redirect |
+| Login page | Done | Google Sign-In button with error display |
+| Security hardening | Done | OAuth state, input validation, .dockerignore |
+| Docker deployment | Done | env_file configuration, production settings |
+
+### Not Yet Implemented
+
+| Feature | Priority | Notes |
+|---------|----------|-------|
+| Admin dashboard | Low | User management, activity viewer |
+| Apple Sign-In | Future | Requires Apple Developer account |
+| Partial public access | Future | Blurred preview for unauthenticated users |
+
+---
+
+## 2. Authentication Scope
+
+### What Requires Authentication
+
+**All pages and API endpoints require authentication.** Unauthenticated users see only the login page.
+
+| Page/Feature | Unauthenticated | Authenticated |
+|--------------|-----------------|---------------|
+| Login Page (`/login`) | Accessible | Redirects to `/` |
+| Overview Page (`/`) | Redirect to login | Fully accessible |
+| Units Page (`/units`) | Redirect to login | Fully accessible |
+| Analytics Page (`/analytics`) | Redirect to login | Fully accessible |
+| All API Endpoints | 401 Unauthorized | Accessible |
+
+### Protected API Endpoints
+
+```
+GET /api/health               → Public (health check only)
+GET /api/daily-summary        → Protected
+GET /api/price-history        → Protected
+GET /api/available-units      → Protected
+GET /api/recent-activity      → Protected
+GET /api/plan-stats           → Protected
+GET /api/unit/:code/history   → Protected
+GET /api/analytics            → Protected
+GET /api/best-deals           → Protected
+GET /api/price-drops          → Protected
+GET /api/stale-inventory      → Protected
+GET /api/market-insights      → Protected
+```
+
+### Auth Routes
+
+```
+GET  /auth/google             → Initiates OAuth flow (sets state cookie)
+GET  /auth/google/callback    → Handles callback (validates state, sets JWT)
+GET  /auth/me                 → Returns current user info
+GET  /auth/status             → Returns { authenticated: true/false }
+POST /auth/logout             → Clears auth cookie
+```
+
+---
+
+## 3. Technical Architecture
+
+### Authentication Flow
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                           Google OAuth Flow                                  │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│  1. User clicks "Sign in with Google"                                       │
+│           │                                                                 │
+│           ▼                                                                 │
+│  2. Frontend redirects to: /api/auth/google                                 │
+│           │                                                                 │
+│           ▼                                                                 │
+│  3. Backend generates state parameter, stores in cookie                     │
+│           │                                                                 │
+│           ▼                                                                 │
+│  4. Backend redirects to Google's OAuth consent screen                      │
+│           │                                                                 │
+│           ▼                                                                 │
+│  5. User authenticates with Google                                          │
+│           │                                                                 │
+│           ▼                                                                 │
+│  6. Google redirects to: /api/auth/google/callback?code=XXX&state=YYY       │
+│           │                                                                 │
+│           ▼                                                                 │
+│  7. Backend validates state parameter against cookie (CSRF protection)      │
+│           │                                                                 │
+│           ▼                                                                 │
+│  8. Backend exchanges code for tokens, fetches user profile                 │
+│           │                                                                 │
+│           ▼                                                                 │
+│  9. Backend creates/updates user in MongoDB (upsert)                        │
+│           │                                                                 │
+│           ▼                                                                 │
+│  10. Backend creates JWT, sets HTTP-only cookie                             │
+│           │                                                                 │
+│           ▼                                                                 │
+│  11. Backend redirects to frontend                                          │
+│           │                                                                 │
+│           ▼                                                                 │
+│  12. Frontend AuthContext checks /auth/status, renders authenticated UI     │
+│                                                                             │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+### Token Refresh Strategy (Sliding Window)
+
+Active users get their tokens refreshed automatically to avoid abrupt logouts.
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                        Sliding Window Refresh                            │
+├─────────────────────────────────────────────────────────────────────────┤
+│                                                                         │
+│  1. User makes authenticated request                                    │
+│           │                                                             │
+│           ▼                                                             │
+│  2. Auth middleware validates JWT                                       │
+│           │                                                             │
+│           ▼                                                             │
+│  3. Check: Is user still active? (isActive field)                       │
+│           │                                                             │
+│      ┌────┴────┐                                                        │
+│      │         │                                                        │
+│    No ▼       Yes ▼                                                     │
+│  Clear cookie  4. Check token age: (now - iat) > 1 day?                 │
+│  Return 401         │                                                   │
+│                ┌────┴────┐                                              │
+│                │         │                                              │
+│              No ▼       Yes ▼                                           │
+│           Continue    5. Issue new JWT with fresh 7-day expiry          │
+│           normally         │                                            │
+│                            ▼                                            │
+│                       6. Set new cookie in response                     │
+│                                                                         │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+| Scenario | Token Age | Action |
+|----------|-----------|--------|
+| User active daily | < 1 day since last refresh | No refresh |
+| User returns after 2 days | 2 days old | Refresh token |
+| User returns after 8 days | Expired | 401, redirect to login |
+
+---
+
+## 4. Implementation Details
+
+### Auth Configuration (`config/auth.js`)
+
+```javascript
+module.exports = {
+  google: {
+    clientID: process.env.GOOGLE_CLIENT_ID,
+    clientSecret: process.env.GOOGLE_CLIENT_SECRET,
+    callbackURL: process.env.GOOGLE_CALLBACK_URL,
+    scope: ['profile', 'email']
+  },
+  jwt: {
+    secret: process.env.JWT_SECRET,
+    expiresIn: '7d',
+    refreshThreshold: 24 * 60 * 60  // 1 day in seconds
+  },
+  cookie: {
+    name: 'auth_token',
+    options: {
+      httpOnly: true,
+      secure: process.env.NODE_ENV === 'production',
+      sameSite: 'lax',
+      maxAge: 7 * 24 * 60 * 60 * 1000,  // 7 days
+      domain: process.env.NODE_ENV === 'production'
+        ? '.maverickapplications.com'
+        : undefined
+    }
+  }
+};
+```
+
+### User Model (`models/user.js`)
+
+Uses MongoDB native driver with upsert pattern:
+
+```javascript
+async function findOrCreateUser(db, profile) {
+  const now = new Date();
+
+  const result = await db.collection('users').findOneAndUpdate(
+    { googleId: profile.googleId },
+    {
+      $set: {
+        email: profile.email,
+        name: profile.name,
+        picture: profile.picture || null,
+        lastLoginAt: now
+      },
+      $inc: { loginCount: 1 },
+      $setOnInsert: {
+        googleId: profile.googleId,
+        isActive: true,
+        role: 'user',
+        createdAt: now
+      }
+    },
+    { upsert: true, returnDocument: 'after' }
+  );
+
+  return result;
+}
+```
+
+**Note:** `loginCount` must NOT be in `$setOnInsert` - it conflicts with `$inc`.
+
+### Passport Strategy (`middleware/passport.js`)
+
+Safely extracts profile data from Google:
+
+```javascript
+async (accessToken, refreshToken, profile, done) => {
+  try {
+    const email = profile.emails?.[0]?.value;
+    if (!email) {
+      return done(new Error('No email found in Google profile'), null);
+    }
+
+    const userProfile = {
+      googleId: profile.id,
+      email: email,
+      name: profile.displayName,
+      picture: profile.photos?.[0]?.value || null
+    };
+
+    const user = await findOrCreateUser(db, userProfile);
+    done(null, user);
+  } catch (error) {
+    console.error('Passport strategy error:', error);
+    done(error, null);
+  }
+}
+```
+
+### OAuth State Validation (`routes/auth.js`)
+
+CSRF protection using state parameter:
+
+```javascript
+// Initiate OAuth - generate and store state
+router.get('/google', (req, res, next) => {
+  const state = crypto.randomBytes(32).toString('hex');
+  res.cookie('oauth_state', state, {
+    httpOnly: true,
+    secure: process.env.NODE_ENV === 'production',
+    sameSite: 'lax',
+    maxAge: 5 * 60 * 1000  // 5 minutes
+  });
+  passport.authenticate('google', {
+    scope: authConfig.google.scope,
+    session: false,
+    state: state
+  })(req, res, next);
+});
+
+// Callback - validate state before processing
+router.get('/google/callback', (req, res, next) => {
+  const stateFromCookie = req.cookies.oauth_state;
+  const stateFromQuery = req.query.state;
+  res.clearCookie('oauth_state');
+
+  if (!stateFromCookie || !stateFromQuery || stateFromCookie !== stateFromQuery) {
+    return res.redirect(`${process.env.FRONTEND_URL}/login?error=invalid_state`);
+  }
+  // ... proceed with authentication
+});
+```
+
+### Frontend Auth Context (`context/AuthContext.jsx`)
+
+```javascript
+const AuthContext = createContext(null);
+
+export function AuthProvider({ children }) {
+  const [user, setUser] = useState(null);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    checkAuthStatus();
+  }, []);
+
+  const checkAuthStatus = async () => {
+    try {
+      const response = await fetch(`${API_BASE}/auth/status`, {
+        credentials: 'include'
+      });
+      const data = await response.json();
+      if (data.authenticated) {
+        setUser(data.user);
+      }
+    } catch (error) {
+      console.error('Auth check failed:', error);
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  const login = () => {
+    window.location.href = `${API_BASE}/auth/google`;
+  };
+
+  const logout = async () => {
+    await fetch(`${API_BASE}/auth/logout`, {
+      method: 'POST',
+      credentials: 'include'
+    });
+    setUser(null);
+  };
+
+  return (
+    <AuthContext.Provider value={{ user, loading, login, logout }}>
+      {children}
+    </AuthContext.Provider>
+  );
+}
+```
+
+### Database Indexes
+
+Created automatically on server startup:
+
+**Users Collection:**
+- `{ googleId: 1 }` - unique
+- `{ email: 1 }` - unique
+- `{ isActive: 1, lastLoginAt: -1 }` - compound for queries
+
+**User Activity Collection:**
+- `{ userId: 1, timestamp: -1 }` - user activity queries
+- `{ timestamp: 1 }` - TTL index (90-day auto-cleanup)
+- `{ action: 1, timestamp: -1 }` - action type queries
+- `{ sessionId: 1 }` - session grouping
+
+---
+
+## 5. User Activity Logging
+
+### Configuration
+
+Controlled by `ACTIVITY_LOG_LEVEL` environment variable:
+
+| Level | What's Logged |
+|-------|---------------|
+| `all` | All user interactions (default) |
+| `navigation` | Only page views and route changes |
+| `none` | Logging disabled |
+
+### Activity Types
+
+| Action | When Logged | Metadata |
+|--------|-------------|----------|
+| `LOGIN` | User completes OAuth | `{ method: 'google' }` |
+| `LOGOUT` | User logs out | `{}` |
+| `PAGE_VIEW` | Page load | `{ path }` |
+| `NAVIGATION` | Route change | `{ from, to }` |
+
+### Activity Schema
+
+```javascript
+{
+  _id: ObjectId,
+  userId: ObjectId,
+  sessionId: String,      // UUID for grouping
+  action: String,
+  metadata: Object,
+  page: String,
+  timestamp: Date,
+  userAgent: String,
+  ip: String
+}
+```
+
+### Auto-Cleanup
+
+Activity records are automatically deleted after 90 days via MongoDB TTL index.
+
+---
+
+## 6. Security Measures
+
+### Implemented
+
+| Security Feature | Implementation |
+|------------------|----------------|
+| OAuth state parameter | Random 32-byte hex, stored in cookie, validated on callback |
+| HTTP-only cookies | JWT not accessible via JavaScript |
+| Secure cookies (production) | Only sent over HTTPS |
+| SameSite=Lax | CSRF protection for cookies |
+| Input validation | Unit codes validated with regex pattern |
+| isActive check | Disabled users rejected on every request |
+| .dockerignore | Prevents .env and secrets from being copied into images |
+| Environment validation | Server exits if MONGO_URI missing in production |
+
+### Input Validation
+
+```javascript
+const isValidUnitCode = (unitCode) => {
+  return typeof unitCode === 'string' &&
+         unitCode.length > 0 &&
+         unitCode.length <= 20 &&
+         /^[A-Za-z0-9_-]+$/.test(unitCode);
+};
+```
+
+### CORS Configuration
+
+```javascript
+const corsOptions = {
+  origin: process.env.FRONTEND_URL,  // Exact origin, not '*'
+  credentials: true,                  // Required for cookies
+  methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
+  allowedHeaders: ['Content-Type', 'Authorization'],
+  exposedHeaders: ['set-cookie']
+};
+```
+
+---
+
+## 7. Deployment Configuration
+
+### Environment Variables
+
+Backend `.env` (see `.env.example` for template):
+
+```env
+# MongoDB
+MONGO_URI=mongodb+srv://...
+
+# Server
+PORT=8080
+NODE_ENV=production
+
+# Google OAuth
+GOOGLE_CLIENT_ID=xxx.apps.googleusercontent.com
+GOOGLE_CLIENT_SECRET=xxx
+GOOGLE_CALLBACK_URL=https://apartments.maverickapplications.com/api/auth/google/callback
+
+# JWT (generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))")
+JWT_SECRET=your-secure-random-string
+
+# App
+FRONTEND_URL=https://apartments.maverickapplications.com
+
+# Activity Logging
+ACTIVITY_LOG_LEVEL=all
+```
+
+### Docker Compose
+
+```yaml
+services:
+  apartment-api:
+    build: .
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=production
+```
+
+### Google Cloud Console Setup
+
+Required redirect URIs:
+```
+https://apartments.maverickapplications.com/api/auth/google/callback
+```
+
+OAuth consent screen:
+- User type: External
+- Scopes: `email`, `profile` (non-sensitive)
+
+---
+
+## 8. Test Checklist
+
+### Completed Tests
+
+- [x] Google OAuth login flow works end-to-end
+- [x] JWT cookie is set with correct flags (httpOnly, secure, sameSite)
+- [x] Protected endpoints return 401 without valid token
+- [x] User data persists across sessions (cookie survives browser close)
+- [x] Activity logging captures LOGIN/LOGOUT events
+- [x] Data loads correctly regardless of server timezone
+- [x] OAuth state parameter prevents CSRF
+- [x] User upsert creates new users and updates existing
+- [x] Sliding window token refresh extends sessions
+- [x] Logout clears cookie and redirects to login
+
+### Production Verification
+
+- [x] OAuth redirect URIs match production domain
+- [x] HTTPS enforced
+- [x] Environment variables properly set
+- [x] MongoDB indexes created on startup
+- [x] No sensitive data in console logs
+- [x] .dockerignore prevents secrets in image
+
+---
+
+## 9. Future: Admin Dashboard
+
+**Priority: Low**
+
+### Planned Features
+
+1. **User Management**
+   - View all registered users
+   - See last login, login count
+   - Enable/disable users
+
+2. **Activity Viewer**
+   - Recent activity stream
+   - Filter by user, action, date
+
+3. **Usage Statistics**
+   - Active users (daily/weekly/monthly)
+   - Most common actions
+   - Peak usage times
+
+### Access Control
+
+- Add `role: 'admin'` to user document
+- Create `requireAdmin` middleware
+- Manually set initial admin via database
+
+---
+
+## 10. Future: Adding Apple Sign-In
+
+**Priority: Future**
+
+### Requirements
+
+1. Apple Developer account ($99/year)
+2. Service ID for web authentication
+3. Private key for token verification
+
+### Changes Needed
+
+1. Install `passport-apple`
+2. Add Apple strategy to passport
+3. Add routes: `/auth/apple`, `/auth/apple/callback`
+4. Update user model for multiple providers
+5. Add "Sign in with Apple" button
+
+---
+
+## 11. Future: Partial Public Access
+
+**Priority: Future**
+
+Allow unauthenticated users to see a blurred preview of the overview page.
+
+### Behavior
+
+- Summary cards visible
+- Charts and detailed data blurred with CSS
+- Overlay with "Sign in to view" prompt
+
+### Implementation
+
+1. Make `/api/daily-summary` public
+2. Create `AuthBlurOverlay` component
+3. Wrap protected sections on overview page
+
+---
+
+## Decisions Log
+
+| Decision | Choice | Rationale |
+|----------|--------|-----------|
+| Token storage | HTTP-only cookie | More secure than localStorage |
+| Token expiry | 7 days with sliding refresh | Balance security and UX |
+| Disabled user handling | Silent logout | No error messaging needed |
+| State parameter | Random 32-byte hex | Industry standard CSRF protection |
+| Activity cleanup | 90-day TTL | Automatic via MongoDB index |
+| Date handling | Use latest database date | Handles server timezone mismatch |
+| User upsert | findOneAndUpdate | Atomic create/update operation |
+
+---
+
+## Troubleshooting
+
+### Common Issues
+
+| Issue | Cause | Solution |
+|-------|-------|----------|
+| 401 on all endpoints | Missing/invalid JWT | Check cookie is set, not expired |
+| OAuth callback 500 | Profile field access error | Use optional chaining on profile fields |
+| Empty data arrays | Timezone mismatch | Server uses latest date from database |
+| MongoDB conflict error | loginCount in $setOnInsert | Remove from $setOnInsert, use only $inc |
+| Cookie not set | CORS misconfiguration | Ensure credentials: 'include' on all fetches |
+| State validation fails | Cookie expired or cleared | State cookie has 5-minute lifetime |

Dockerfile

@@ -1,8 +1,12 @@
-FROM node:18-alpine
+FROM node:20-alpine
 
 # Set working directory
 WORKDIR /app
 
+# Update Alpine packages and npm to fix security vulnerabilities
+RUN apk upgrade --no-cache && \
+    npm install -g npm@latest
+
 # Install Curl
 RUN apk add --no-cache curl
 
@@ -10,7 +14,7 @@ RUN apk add --no-cache curl
 COPY package*.json ./
 
 # Install dependencies
-RUN npm ci --only=production
+RUN npm ci --omit=dev
 
 # Copy source code
 COPY . .

__tests__/helpers/testHelpers.js

@@ -0,0 +1,156 @@
+const { MongoClient, ObjectId } = require('mongodb');
+const jwt = require('jsonwebtoken');
+
+// Counter for generating unique identifiers within the same millisecond
+let uniqueCounter = 0;
+
+/**
+ * Create a test Express app instance with database connection
+ * This creates a minimal Express app for testing admin routes
+ */
+async function createTestApp(db) {
+  const express = require('express');
+  const cookieParser = require('cookie-parser');
+
+  const app = express();
+  app.use(express.json());
+  app.use(cookieParser());
+
+  // Store db in app.locals for middleware access
+  app.locals.db = db;
+
+  // Mount the auth middleware module
+  const { requireAuth, requireAdmin } = require('../../middleware/auth');
+
+  // Health check endpoint (should remain public)
+  app.get('/api/health', (req, res) => {
+    res.json({ status: 'ok' });
+  });
+
+  // Try to mount admin routes if they exist
+  try {
+    const adminRoutes = require('../../routes/admin');
+    app.use('/api/admin', requireAuth, requireAdmin, adminRoutes);
+  } catch (error) {
+    // Admin routes don't exist yet - this is expected for failing tests
+    // Create placeholder routes that will 404
+    app.use('/api/admin', requireAuth, requireAdmin, (req, res) => {
+      res.status(404).json({ error: 'Admin routes not implemented' });
+    });
+  }
+
+  return app;
+}
+
+/**
+ * Generate a valid JWT token for a test user
+ * @param {string|ObjectId} userId - The user's MongoDB _id
+ * @returns {string} JWT token
+ */
+function generateTestToken(userId) {
+  const secret = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
+  return jwt.sign(
+    { userId: userId.toString() },
+    secret,
+    { expiresIn: '7d' }
+  );
+}
+
+/**
+ * Create a test user document
+ * @param {Object} overrides - Fields to override in the default user
+ * @returns {Object} User document
+ */
+function createTestUser(overrides = {}) {
+  const now = new Date();
+  const uniqueId = `${Date.now()}-${++uniqueCounter}-${Math.random().toString(36).substr(2, 9)}`;
+  return {
+    _id: new ObjectId(),
+    googleId: `google-${uniqueId}`,
+    email: `test-${uniqueId}@example.com`,
+    name: 'Test User',
+    picture: 'https://example.com/photo.jpg',
+    role: 'user',
+    isActive: true,
+    loginCount: 1,
+    createdAt: now,
+    lastLoginAt: now,
+    disabledAt: null,
+    disabledBy: null,
+    ...overrides
+  };
+}
+
+/**
+ * Create an admin user document
+ * @param {Object} overrides - Fields to override in the default admin
+ * @returns {Object} Admin user document
+ */
+function createTestAdmin(overrides = {}) {
+  const uniqueId = `${Date.now()}-${++uniqueCounter}-${Math.random().toString(36).substr(2, 9)}`;
+  return createTestUser({
+    role: 'admin',
+    email: `admin-${uniqueId}@example.com`,
+    name: 'Test Admin',
+    ...overrides
+  });
+}
+
+/**
+ * Insert a user into the database
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} user - User document to insert
+ * @returns {Promise<Object>} Inserted user with _id
+ */
+async function insertTestUser(db, user) {
+  const result = await db.collection('users').insertOne(user);
+  return { ...user, _id: result.insertedId };
+}
+
+/**
+ * Clean up test users from the database
+ * @param {Db} db - MongoDB database instance
+ */
+async function cleanupTestUsers(db) {
+  await db.collection('users').deleteMany({});
+}
+
+/**
+ * Clean up all test data from the database
+ * @param {Db} db - MongoDB database instance
+ */
+async function cleanupTestData(db) {
+  await Promise.all([
+    db.collection('users').deleteMany({}),
+    db.collection('user_activity').deleteMany({})
+  ]);
+}
+
+/**
+ * Create a test activity document
+ * @param {Object} overrides - Fields to override in the default activity
+ * @returns {Object} Activity document
+ */
+function createTestActivity(overrides = {}) {
+  const now = new Date();
+  return {
+    _id: new ObjectId(),
+    userId: new ObjectId(),
+    action: 'PAGE_VIEW',
+    metadata: { path: '/test' },
+    timestamp: now,
+    sessionId: `session-${Date.now()}`,
+    ...overrides
+  };
+}
+
+module.exports = {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  insertTestUser,
+  cleanupTestUsers,
+  cleanupTestData,
+  createTestActivity
+};

__tests__/phase1-foundation.test.js

@@ -0,0 +1,905 @@
+/**
+ * Phase 1: Foundation Tests for Admin Dashboard
+ *
+ * These tests verify the implementation of Phase 1 requirements from ADMIN.md:
+ * - 1.1 Database Schema (DB-1.x): User role field, disabledAt/disabledBy fields, indexes
+ * - 1.2 Middleware (MW-1.x): requireAdmin middleware functionality
+ * - 1.3 User Management API (API-1.x): CRUD operations for admin user management
+ *
+ * These tests are designed to FAIL initially as the implementation does not exist yet.
+ * Run with: npm test
+ */
+
+const request = require('supertest');
+const { MongoClient, ObjectId } = require('mongodb');
+const {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  insertTestUser,
+  cleanupTestUsers,
+  cleanupTestData
+} = require('./helpers/testHelpers');
+
+describe('Phase 1: Foundation', () => {
+  let connection;
+  let db;
+  let app;
+
+  beforeAll(async () => {
+    // Connect to the in-memory MongoDB instance
+    const uri = process.env.MONGO_URI;
+    connection = await MongoClient.connect(uri);
+    db = connection.db('apartments_test');
+  });
+
+  afterAll(async () => {
+    if (connection) {
+      await connection.close();
+    }
+  });
+
+  beforeEach(async () => {
+    // Clean up test data before each test
+    await cleanupTestData(db);
+    // Create fresh app instance for each test
+    app = await createTestApp(db);
+  });
+
+  // =============================================================================
+  // 1.1 DATABASE SCHEMA TESTS
+  // These tests verify the user schema has been properly updated for admin features
+  // =============================================================================
+  describe('1.1 Database Schema', () => {
+    /**
+     * DB-1.1: Users must have a `role` field
+     * Valid values: 'user' | 'admin'
+     * Default: 'user'
+     */
+    describe('DB-1.1: User role field', () => {
+      it('should have role field with default value "user" for new users', async () => {
+        // When a new user is created without specifying role
+        const user = createTestUser();
+        delete user.role; // Remove role to test default
+
+        const { findOrCreateUser } = require('../models/user');
+
+        // Create a user profile to simulate OAuth flow
+        const profile = {
+          googleId: user.googleId,
+          email: user.email,
+          name: user.name,
+          picture: user.picture
+        };
+
+        const createdUser = await findOrCreateUser(db, profile);
+
+        // Then the role should default to 'user'
+        expect(createdUser).toBeDefined();
+        expect(createdUser.role).toBe('user');
+      });
+
+      it('should only allow "user" or "admin" as valid role values', async () => {
+        // Attempt to insert a user with an invalid role
+        const userWithInvalidRole = createTestUser({ role: 'superadmin' });
+
+        // This test verifies schema validation exists
+        // The exact implementation depends on whether validation is done at
+        // the application level or database level
+        await expect(async () => {
+          await db.collection('users').insertOne(userWithInvalidRole);
+          // Query the user back to verify role validation
+          const inserted = await db.collection('users').findOne({ _id: userWithInvalidRole._id });
+          // If no validation exists, this should be caught by application logic
+          if (inserted.role !== 'user' && inserted.role !== 'admin') {
+            throw new Error('Invalid role should not be allowed');
+          }
+        }).rejects.toThrow();
+      });
+    });
+
+    /**
+     * DB-1.2: Users must have `disabledAt` field
+     * Type: Date | null
+     * Set when user is disabled, null when active
+     */
+    describe('DB-1.2: User disabledAt field', () => {
+      it('should have disabledAt field set to null for active users', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        const foundUser = await db.collection('users').findOne({ _id: user._id });
+
+        expect(foundUser.disabledAt).toBeNull();
+      });
+
+      it('should have disabledAt field set to Date when user is disabled', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        // Import the user model function that handles disabling
+        const { setUserActive } = require('../models/user');
+
+        // Disable the user (this should set disabledAt)
+        const disabledUser = await setUserActive(db, user._id, false);
+
+        expect(disabledUser.isActive).toBe(false);
+        expect(disabledUser.disabledAt).toBeInstanceOf(Date);
+      });
+    });
+
+    /**
+     * DB-1.3: Users must have `disabledBy` field
+     * Type: ObjectId | null
+     * References the admin who disabled the user
+     */
+    describe('DB-1.3: User disabledBy field', () => {
+      it('should have disabledBy field set to null for active users', async () => {
+        const user = createTestUser({ isActive: true });
+        await insertTestUser(db, user);
+
+        const foundUser = await db.collection('users').findOne({ _id: user._id });
+
+        expect(foundUser.disabledBy).toBeNull();
+      });
+
+      it('should have disabledBy field set to admin ObjectId when disabled', async () => {
+        const admin = createTestAdmin();
+        const user = createTestUser();
+        await insertTestUser(db, admin);
+        await insertTestUser(db, user);
+
+        // Import the updated user model function that accepts disabledBy
+        const { setUserActive } = require('../models/user');
+
+        // Disable the user with admin reference
+        const disabledUser = await setUserActive(db, user._id, false, admin._id);
+
+        expect(disabledUser.isActive).toBe(false);
+        expect(disabledUser.disabledBy).toEqual(admin._id);
+      });
+    });
+
+    /**
+     * DB-1.4: Required indexes for admin queries
+     * Indexes needed:
+     * - { role: 1 }
+     * - { isActive: 1, role: 1 }
+     * - { createdAt: -1 }
+     * - { lastLoginAt: -1 }
+     */
+    describe('DB-1.4: Required indexes', () => {
+      beforeEach(async () => {
+        // Create indexes (this should be done by createIndexes function)
+        const { createIndexes } = require('../models/user');
+        await createIndexes(db);
+      });
+
+      it('should have index on role field', async () => {
+        const indexes = await db.collection('users').indexes();
+        const roleIndex = indexes.find(idx =>
+          idx.key && idx.key.role === 1 && Object.keys(idx.key).length === 1
+        );
+
+        expect(roleIndex).toBeDefined();
+      });
+
+      it('should have compound index on isActive and role', async () => {
+        const indexes = await db.collection('users').indexes();
+        const compoundIndex = indexes.find(idx =>
+          idx.key && idx.key.isActive === 1 && idx.key.role === 1
+        );
+
+        expect(compoundIndex).toBeDefined();
+      });
+
+      it('should have index on createdAt descending', async () => {
+        const indexes = await db.collection('users').indexes();
+        const createdAtIndex = indexes.find(idx =>
+          idx.key && idx.key.createdAt === -1
+        );
+
+        expect(createdAtIndex).toBeDefined();
+      });
+
+      it('should have index on lastLoginAt descending', async () => {
+        const indexes = await db.collection('users').indexes();
+        const lastLoginIndex = indexes.find(idx =>
+          idx.key && idx.key.lastLoginAt === -1
+        );
+
+        expect(lastLoginIndex).toBeDefined();
+      });
+    });
+  });
+
+  // =============================================================================
+  // 1.2 MIDDLEWARE TESTS
+  // These tests verify the requireAdmin middleware behaves correctly
+  // =============================================================================
+  describe('1.2 Middleware', () => {
+    /**
+     * MW-1.1: requireAdmin middleware must exist
+     */
+    describe('MW-1.1: requireAdmin middleware exists', () => {
+      it('should export requireAdmin middleware from auth module', () => {
+        const { requireAdmin } = require('../middleware/auth');
+
+        expect(requireAdmin).toBeDefined();
+        expect(typeof requireAdmin).toBe('function');
+      });
+    });
+
+    /**
+     * MW-1.2: requireAdmin returns 403 if user.role !== 'admin'
+     */
+    describe('MW-1.2: requireAdmin authorization', () => {
+      it('should return 403 with "Admin access required" for non-admin users', async () => {
+        // Create a regular user (not admin)
+        const user = createTestUser({ role: 'user' });
+        await insertTestUser(db, user);
+        const token = generateTestToken(user._id);
+
+        // Attempt to access an admin endpoint
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(403);
+        expect(response.body.error).toBe('Admin access required');
+      });
+
+      it('should allow access for users with admin role', async () => {
+        // Create an admin user
+        const admin = createTestAdmin();
+        await insertTestUser(db, admin);
+        const token = generateTestToken(admin._id);
+
+        // Access admin endpoint - should not get 403
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        // Should either succeed (200) or 404 if routes not implemented
+        // but NOT 403 (forbidden)
+        expect(response.status).not.toBe(403);
+      });
+    });
+
+    /**
+     * MW-1.3: requireAdmin must work after requireAuth
+     */
+    describe('MW-1.3: Middleware chaining', () => {
+      it('should return 401 if no authentication token provided', async () => {
+        const response = await request(app)
+          .get('/api/admin/users');
+
+        expect(response.status).toBe(401);
+        expect(response.body.error).toBe('Authentication required');
+      });
+
+      it('should return 401 for invalid token before checking admin role', async () => {
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', ['auth_token=invalid-token']);
+
+        expect(response.status).toBe(401);
+      });
+
+      it('should return 401 for disabled users even if they have admin role', async () => {
+        // Create a disabled admin
+        const disabledAdmin = createTestAdmin({ isActive: false });
+        await insertTestUser(db, disabledAdmin);
+        const token = generateTestToken(disabledAdmin._id);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(401);
+      });
+    });
+
+    /**
+     * MW-1.4: /api/health must remain public
+     */
+    describe('MW-1.4: Health endpoint remains public', () => {
+      it('should return 200 on /api/health without authentication', async () => {
+        const response = await request(app)
+          .get('/api/health');
+
+        expect(response.status).toBe(200);
+        expect(response.body.status).toBe('ok');
+      });
+
+      it('should not require admin role for /api/health', async () => {
+        // Regular user accessing health endpoint
+        const user = createTestUser({ role: 'user' });
+        await insertTestUser(db, user);
+        const token = generateTestToken(user._id);
+
+        const response = await request(app)
+          .get('/api/health')
+          .set('Cookie', [`auth_token=${token}`]);
+
+        expect(response.status).toBe(200);
+      });
+    });
+  });
+
+  // =============================================================================
+  // 1.3 USER MANAGEMENT API TESTS
+  // These tests verify the admin user management endpoints
+  // =============================================================================
+  describe('1.3 User Management API', () => {
+    let adminUser;
+    let adminToken;
+
+    beforeEach(async () => {
+      // Create an admin user for testing admin endpoints
+      adminUser = createTestAdmin();
+      await insertTestUser(db, adminUser);
+      adminToken = generateTestToken(adminUser._id);
+    });
+
+    /**
+     * API-1.1: GET /api/admin/users - Paginated user list
+     */
+    describe('API-1.1: GET /api/admin/users', () => {
+      it('should return paginated list of users', async () => {
+        // Create some test users
+        for (let i = 0; i < 25; i++) {
+          await insertTestUser(db, createTestUser({
+            email: `user${i}@example.com`,
+            name: `User ${i}`
+          }));
+        }
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users).toBeDefined();
+        expect(Array.isArray(response.body.data.users)).toBe(true);
+        expect(response.body.data.pagination).toBeDefined();
+        expect(response.body.data.pagination.page).toBe(1);
+        expect(response.body.data.pagination.limit).toBe(20);
+        expect(response.body.data.pagination.total).toBeGreaterThan(0);
+        expect(response.body.data.pagination.pages).toBeDefined();
+      });
+
+      it('should support page parameter', async () => {
+        // Create 30 test users
+        for (let i = 0; i < 30; i++) {
+          await insertTestUser(db, createTestUser({
+            email: `user${i}@example.com`
+          }));
+        }
+
+        const response = await request(app)
+          .get('/api/admin/users?page=2')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.page).toBe(2);
+      });
+
+      it('should support limit parameter with max 100', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?limit=50')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.limit).toBe(50);
+      });
+
+      it('should cap limit at 100', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?limit=200')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.pagination.limit).toBeLessThanOrEqual(100);
+      });
+
+      it('should support search parameter for name', async () => {
+        await insertTestUser(db, createTestUser({ name: 'John Smith' }));
+        await insertTestUser(db, createTestUser({ name: 'Jane Doe' }));
+
+        const response = await request(app)
+          .get('/api/admin/users?search=John')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.some(u => u.name.includes('John'))).toBe(true);
+        expect(response.body.data.users.every(u =>
+          u.name.toLowerCase().includes('john') ||
+          u.email.toLowerCase().includes('john')
+        )).toBe(true);
+      });
+
+      it('should support search parameter for email', async () => {
+        await insertTestUser(db, createTestUser({ email: 'unique-test@example.com' }));
+
+        const response = await request(app)
+          .get('/api/admin/users?search=unique-test')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.some(u => u.email.includes('unique-test'))).toBe(true);
+      });
+
+      it('should support status filter "active"', async () => {
+        await insertTestUser(db, createTestUser({ isActive: true }));
+        await insertTestUser(db, createTestUser({ isActive: false }));
+
+        const response = await request(app)
+          .get('/api/admin/users?status=active')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.every(u => u.isActive === true)).toBe(true);
+      });
+
+      it('should support status filter "disabled"', async () => {
+        await insertTestUser(db, createTestUser({ isActive: false }));
+
+        const response = await request(app)
+          .get('/api/admin/users?status=disabled')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.users.every(u => u.isActive === false)).toBe(true);
+      });
+
+      it('should support sort parameter', async () => {
+        const oldUser = createTestUser({
+          createdAt: new Date('2023-01-01'),
+          email: 'old@example.com'
+        });
+        const newUser = createTestUser({
+          createdAt: new Date('2024-01-01'),
+          email: 'new@example.com'
+        });
+        await insertTestUser(db, oldUser);
+        await insertTestUser(db, newUser);
+
+        const response = await request(app)
+          .get('/api/admin/users?sort=createdAt&order=desc')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        // Newest first when sorted descending
+        const createdDates = response.body.data.users.map(u => new Date(u.createdAt));
+        for (let i = 0; i < createdDates.length - 1; i++) {
+          expect(createdDates[i] >= createdDates[i + 1]).toBe(true);
+        }
+      });
+
+      it('should support order parameter "asc" and "desc"', async () => {
+        const response = await request(app)
+          .get('/api/admin/users?sort=loginCount&order=asc')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        // Verify ascending order
+        const counts = response.body.data.users.map(u => u.loginCount);
+        for (let i = 0; i < counts.length - 1; i++) {
+          expect(counts[i] <= counts[i + 1]).toBe(true);
+        }
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, regularUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${userToken}`]);
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.2: GET /api/admin/users/:id - User details with recent activity
+     */
+    describe('API-1.2: GET /api/admin/users/:id', () => {
+      it('should return user details for valid user ID', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user._id.toString()).toBe(testUser._id.toString());
+        expect(response.body.data.user.email).toBe(testUser.email);
+        expect(response.body.data.user.name).toBe(testUser.name);
+        expect(response.body.data.user.role).toBe(testUser.role);
+      });
+
+      it('should include recent activity for the user', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        // Create some activity for this user
+        await db.collection('user_activity').insertMany([
+          {
+            userId: testUser._id,
+            action: 'PAGE_VIEW',
+            metadata: { path: '/dashboard' },
+            timestamp: new Date(),
+            sessionId: 'session-1'
+          },
+          {
+            userId: testUser._id,
+            action: 'LOGIN',
+            timestamp: new Date(),
+            sessionId: 'session-1'
+          }
+        ]);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user.recentActivity).toBeDefined();
+        expect(Array.isArray(response.body.data.user.recentActivity)).toBe(true);
+      });
+
+      it('should return 404 for non-existent user ID', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .get(`/api/admin/users/${nonExistentId}`)
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(404);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should return 400 for invalid user ID format', async () => {
+        const response = await request(app)
+          .get('/api/admin/users/invalid-id')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, regularUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .get(`/api/admin/users/${regularUser._id}`)
+          .set('Cookie', [`auth_token=${userToken}`]);
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.3: PATCH /api/admin/users/:id - Enable/disable user
+     */
+    describe('API-1.3: PATCH /api/admin/users/:id (enable/disable)', () => {
+      it('should disable a user successfully', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.isActive).toBe(false);
+        expect(response.body.data.message).toContain('disabled');
+
+        // Verify in database
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.isActive).toBe(false);
+      });
+
+      it('should enable a disabled user successfully', async () => {
+        const testUser = createTestUser({ isActive: false });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: true });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.isActive).toBe(true);
+        expect(response.body.data.message).toContain('enabled');
+      });
+
+      it('should set disabledAt timestamp when disabling', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const beforeDisable = new Date();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledAt).toBeInstanceOf(Date);
+        expect(dbUser.disabledAt.getTime()).toBeGreaterThanOrEqual(beforeDisable.getTime());
+      });
+
+      it('should set disabledBy to admin ID when disabling', async () => {
+        const testUser = createTestUser({ isActive: true });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledBy).toEqual(adminUser._id);
+      });
+
+      it('should clear disabledAt and disabledBy when enabling', async () => {
+        const testUser = createTestUser({
+          isActive: false,
+          disabledAt: new Date(),
+          disabledBy: new ObjectId()
+        });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: true });
+
+        expect(response.status).toBe(200);
+
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.disabledAt).toBeNull();
+        expect(dbUser.disabledBy).toBeNull();
+      });
+
+      it('should return 400 when trying to disable yourself', async () => {
+        const response = await request(app)
+          .patch(`/api/admin/users/${adminUser._id}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toContain('yourself');
+      });
+
+      it('should return 404 for non-existent user', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${nonExistentId}`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(404);
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        const targetUser = createTestUser();
+        await insertTestUser(db, regularUser);
+        await insertTestUser(db, targetUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${targetUser._id}`)
+          .set('Cookie', [`auth_token=${userToken}`])
+          .send({ isActive: false });
+
+        expect(response.status).toBe(403);
+      });
+    });
+
+    /**
+     * API-1.4: PATCH /api/admin/users/:id/role - Promote/demote user role
+     */
+    describe('API-1.4: PATCH /api/admin/users/:id/role (promote/demote)', () => {
+      it('should promote a user to admin', async () => {
+        const testUser = createTestUser({ role: 'user' });
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.role).toBe('admin');
+        expect(response.body.data.message).toContain('promoted');
+
+        // Verify in database
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(dbUser.role).toBe('admin');
+      });
+
+      it('should demote an admin to user', async () => {
+        const anotherAdmin = createTestAdmin({ email: 'another@admin.com' });
+        await insertTestUser(db, anotherAdmin);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${anotherAdmin._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'user' });
+
+        expect(response.status).toBe(200);
+        expect(response.body.data.user).toBeDefined();
+        expect(response.body.data.user.role).toBe('user');
+        expect(response.body.data.message).toContain('demoted');
+      });
+
+      it('should return 400 when trying to demote yourself from admin', async () => {
+        const response = await request(app)
+          .patch(`/api/admin/users/${adminUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'user' });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toContain('yourself');
+      });
+
+      it('should return 400 for invalid role value', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'superadmin' });
+
+        expect(response.status).toBe(400);
+        expect(response.body.error).toBeDefined();
+      });
+
+      it('should return 400 when role is not provided', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${testUser._id}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({});
+
+        expect(response.status).toBe(400);
+      });
+
+      it('should return 404 for non-existent user', async () => {
+        const nonExistentId = new ObjectId();
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${nonExistentId}/role`)
+          .set('Cookie', [`auth_token=${adminToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(404);
+      });
+
+      it('should require admin role', async () => {
+        const regularUser = createTestUser({ role: 'user' });
+        const targetUser = createTestUser();
+        await insertTestUser(db, regularUser);
+        await insertTestUser(db, targetUser);
+        const userToken = generateTestToken(regularUser._id);
+
+        const response = await request(app)
+          .patch(`/api/admin/users/${targetUser._id}/role`)
+          .set('Cookie', [`auth_token=${userToken}`])
+          .send({ role: 'admin' });
+
+        expect(response.status).toBe(403);
+      });
+
+      it('should only allow "user" or "admin" roles', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const invalidRoles = ['superuser', 'moderator', 'guest', '', null, 123];
+
+        for (const invalidRole of invalidRoles) {
+          const response = await request(app)
+            .patch(`/api/admin/users/${testUser._id}/role`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ role: invalidRole });
+
+          expect(response.status).toBe(400);
+        }
+      });
+    });
+
+    /**
+     * API-1.5: General API security tests
+     */
+    describe('API-1.5: API Security', () => {
+      it('should not expose sensitive user data in list response', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        const response = await request(app)
+          .get('/api/admin/users')
+          .set('Cookie', [`auth_token=${adminToken}`]);
+
+        expect(response.status).toBe(200);
+
+        // Check that sensitive fields are not exposed
+        const userInResponse = response.body.data.users.find(
+          u => u._id.toString() === testUser._id.toString()
+        );
+
+        if (userInResponse) {
+          // googleId should potentially be hidden or sanitized in responses
+          // This depends on your security requirements
+          expect(userInResponse.password).toBeUndefined();
+        }
+      });
+
+      it('should validate ObjectId format in URL parameters', async () => {
+        const invalidIds = ['123', 'abc', '!@#$%', '   ', ''];
+
+        for (const invalidId of invalidIds) {
+          const response = await request(app)
+            .get(`/api/admin/users/${invalidId}`)
+            .set('Cookie', [`auth_token=${adminToken}`]);
+
+          expect(response.status).toBe(400);
+        }
+      });
+
+      it('should handle concurrent requests safely', async () => {
+        const testUser = createTestUser();
+        await insertTestUser(db, testUser);
+
+        // Send multiple concurrent disable/enable requests
+        const requests = [
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: false }),
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: true }),
+          request(app)
+            .patch(`/api/admin/users/${testUser._id}`)
+            .set('Cookie', [`auth_token=${adminToken}`])
+            .send({ isActive: false })
+        ];
+
+        const responses = await Promise.all(requests);
+
+        // All requests should complete without errors (200) or with consistent state
+        responses.forEach(response => {
+          expect([200, 400, 404, 409]).toContain(response.status);
+        });
+
+        // Final state should be consistent
+        const dbUser = await db.collection('users').findOne({ _id: testUser._id });
+        expect(typeof dbUser.isActive).toBe('boolean');
+      });
+    });
+  });
+});

__tests__/phase4-settings-security.test.js

@@ -0,0 +1,980 @@
+/**
+ * Phase 4: Settings & Security Tests
+ *
+ * Tests for Admin Dashboard Phase 4 implementation as specified in ADMIN.md
+ * Reference: Implementation Phases -> Phase 4: Settings & Polish
+ *
+ * Endpoints tested:
+ * - GET /api/admin/settings (API-4.1)
+ * - PATCH /api/admin/settings (API-4.2, API-4.3)
+ *
+ * Security features tested:
+ * - Admin action audit logging (SEC-4.1)
+ * - Last admin protection (SEC-4.2)
+ * - Rate limiting on admin endpoints (SEC-4.3)
+ *
+ * These tests are designed to FAIL initially as the endpoints do not exist yet.
+ * They serve as the specification for implementing the Settings & Polish features.
+ */
+
+const request = require('supertest');
+const { MongoClient, ObjectId } = require('mongodb');
+const {
+  createTestApp,
+  generateTestToken,
+  createTestUser,
+  createTestAdmin,
+  cleanupTestData
+} = require('./helpers/testHelpers');
+
+// Test configuration - uses environment set by global setup
+const TEST_JWT_SECRET = process.env.JWT_SECRET || 'test-jwt-secret-for-testing-only';
+
+let app;
+let db;
+let client;
+
+// Test user fixtures - created fresh for each test
+let testUsers = {};
+
+/**
+ * Helper to make authenticated requests
+ * @param {string} method - HTTP method (get, post, patch, delete)
+ * @param {string} url - Request URL
+ * @param {Object} user - User to authenticate as
+ * @returns {Object} Supertest request
+ */
+const authenticatedRequest = (method, url, user) => {
+  const token = generateTestToken(user._id);
+  return request(app)[method](url)
+    .set('Cookie', `auth_token=${token}`);
+};
+
+/**
+ * Helper object for cleaner authenticated request syntax
+ * @param {Object} user - User to authenticate as
+ * @returns {Object} Object with HTTP method functions
+ */
+const authAs = (user) => ({
+  get: (url) => authenticatedRequest('get', url, user),
+  post: (url) => authenticatedRequest('post', url, user),
+  patch: (url) => authenticatedRequest('patch', url, user),
+  delete: (url) => authenticatedRequest('delete', url, user)
+});
+
+describe('Phase 4: Settings & Security', () => {
+  beforeAll(async () => {
+    // Connect to test database (MongoDB Memory Server from global setup)
+    const mongoUri = process.env.MONGO_URI || 'mongodb://localhost:27017';
+    client = new MongoClient(mongoUri);
+    await client.connect();
+    db = client.db('apartments_test');
+
+    // Create the test app with our database
+    app = await createTestApp(db);
+  });
+
+  afterAll(async () => {
+    if (db) {
+      await cleanupTestData(db);
+      await db.collection('settings').deleteMany({});
+    }
+    if (client) {
+      await client.close();
+    }
+  });
+
+  beforeEach(async () => {
+    // Clean up all test data
+    await cleanupTestData(db);
+    await db.collection('settings').deleteMany({});
+
+    // Create fresh test users with new ObjectIds for each test
+    testUsers = {
+      admin: createTestAdmin({
+        _id: new ObjectId(),
+        email: 'admin@example.com',
+        name: 'Test Admin'
+      }),
+      secondAdmin: createTestAdmin({
+        _id: new ObjectId(),
+        email: 'admin2@example.com',
+        name: 'Second Admin'
+      }),
+      regularUser: createTestUser({
+        _id: new ObjectId(),
+        email: 'user@example.com',
+        name: 'Test User'
+      }),
+      disabledUser: createTestUser({
+        _id: new ObjectId(),
+        email: 'disabled@example.com',
+        name: 'Disabled User',
+        isActive: false
+      })
+    };
+
+    // Insert primary admin and regular user
+    await db.collection('users').insertMany([
+      testUsers.admin,
+      testUsers.regularUser
+    ]);
+
+    // Insert default settings document
+    await db.collection('settings').insertOne({
+      _id: 'admin_settings',
+      activityRetentionDays: 90,
+      activityLogLevel: 'all',
+      updatedAt: new Date(),
+      updatedBy: null
+    });
+  });
+
+  // ============================================================
+  // API-4.1: GET /api/admin/settings - Get admin settings
+  // ============================================================
+  describe('API-4.1: GET /api/admin/settings', () => {
+    describe('Authorization', () => {
+      it('should return 401 when no authentication token is provided', async () => {
+        const response = await request(app)
+          .get('/api/admin/settings')
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 403 when user is not an admin', async () => {
+        const response = await authAs(testUsers.regularUser)
+          .get('/api/admin/settings')
+          .expect(403);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/admin/i);
+      });
+
+      it('should return 401 when admin user is disabled', async () => {
+        // Insert disabled admin
+        const disabledAdmin = createTestAdmin({
+          _id: new ObjectId(),
+          email: 'disabled-admin@example.com',
+          isActive: false
+        });
+        await db.collection('users').insertOne(disabledAdmin);
+
+        const response = await authAs(disabledAdmin)
+          .get('/api/admin/settings')
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 200 when user is an active admin', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(response.body).toHaveProperty('activityRetentionDays');
+        expect(response.body).toHaveProperty('activityLogLevel');
+      });
+    });
+
+    describe('Response format', () => {
+      it('should return activityRetentionDays as a number', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(typeof response.body.activityRetentionDays).toBe('number');
+      });
+
+      it('should return activityLogLevel as a string', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(typeof response.body.activityLogLevel).toBe('string');
+      });
+
+      it('should return default values when no settings document exists', async () => {
+        // Remove settings document
+        await db.collection('settings').deleteMany({});
+
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        // Should return defaults: 90 days retention, 'all' log level
+        expect(response.body.activityRetentionDays).toBe(90);
+        expect(response.body.activityLogLevel).toBe('all');
+      });
+
+      it('should return stored values when settings exist', async () => {
+        // Update settings to non-default values
+        await db.collection('settings').updateOne(
+          { _id: 'admin_settings' },
+          { $set: { activityRetentionDays: 180, activityLogLevel: 'navigation' } }
+        );
+
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/settings')
+          .expect(200);
+
+        expect(response.body.activityRetentionDays).toBe(180);
+        expect(response.body.activityLogLevel).toBe('navigation');
+      });
+    });
+  });
+
+  // ============================================================
+  // API-4.2: PATCH /api/admin/settings - Update settings
+  // ============================================================
+  describe('API-4.2: PATCH /api/admin/settings', () => {
+    describe('Authorization', () => {
+      it('should return 401 when no authentication token is provided', async () => {
+        const response = await request(app)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(401);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 403 when user is not an admin', async () => {
+        const response = await authAs(testUsers.regularUser)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(403);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/admin/i);
+      });
+
+      it('should return 200 when user is an active admin', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 60 })
+          .expect(200);
+
+        expect(response.body.data).toHaveProperty('settings');
+        expect(response.body.data).toHaveProperty('message');
+      });
+    });
+
+    describe('Validation - Retention period (API-4.2)', () => {
+      it('should return 400 for retention period below 30 days', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 29 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/30|365|range|invalid/i);
+      });
+
+      it('should return 400 for retention period above 365 days', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 366 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+        expect(response.body.error).toMatch(/30|365|range|invalid/i);
+      });
+
+      it('should accept retention period at minimum boundary (30 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 30 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(30);
+      });
+
+      it('should accept retention period at maximum boundary (365 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 365 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(365);
+      });
+
+      it('should accept valid retention period within range (180 days)', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 180 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(180);
+      });
+
+      it('should return 400 for non-numeric retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 'ninety' })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 400 for negative retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: -30 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+
+      it('should return 400 for decimal retention period', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 90.5 })
+          .expect(400);
+
+        expect(response.body).toHaveProperty('error');
+      });
+    });
+
+    describe('Successful update', () => {
+      it('should update retention period and return success message', async () => {
+        const response = await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        expect(response.body.data.settings.activityRetentionDays).toBe(120);
+        expect(response.body.data.message).toBe('Settings updated successfully');
+      });
+
+      it('should persist the updated settings in database', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 150 })
+          .expect(200);
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(settings.activityRetentionDays).toBe(150);
+      });
+
+      it('should update updatedAt timestamp', async () => {
+        const beforeUpdate = new Date();
+
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const afterUpdate = new Date();
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(new Date(settings.updatedAt).getTime()).toBeGreaterThanOrEqual(beforeUpdate.getTime());
+        expect(new Date(settings.updatedAt).getTime()).toBeLessThanOrEqual(afterUpdate.getTime());
+      });
+
+      it('should record which admin updated the settings', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const settings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+        expect(settings.updatedBy.toString()).toBe(testUsers.admin._id.toString());
+      });
+    });
+  });
+
+  // ============================================================
+  // API-4.3: TTL Index Update
+  // ============================================================
+  describe('API-4.3: TTL Index Update', () => {
+    it('should update TTL index when retention period changes to 60 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 60 })
+        .expect(200);
+
+      // Check the TTL index on user_activity collection
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 60 days = 60 * 24 * 60 * 60 = 5,184,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(60 * 24 * 60 * 60);
+    });
+
+    it('should update TTL index when retention period changes to 30 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 30 })
+        .expect(200);
+
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 30 days = 2,592,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(30 * 24 * 60 * 60);
+    });
+
+    it('should update TTL index when retention period changes to 365 days', async () => {
+      await authAs(testUsers.admin)
+        .patch('/api/admin/settings')
+        .send({ activityRetentionDays: 365 })
+        .expect(200);
+
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(
+        idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+      );
+
+      expect(ttlIndex).toBeDefined();
+      // 365 days = 31,536,000 seconds
+      expect(ttlIndex.expireAfterSeconds).toBe(365 * 24 * 60 * 60);
+    });
+
+    it('should correctly calculate TTL seconds for various retention periods', async () => {
+      const testCases = [
+        { days: 30, expectedSeconds: 2592000 },
+        { days: 90, expectedSeconds: 7776000 },
+        { days: 180, expectedSeconds: 15552000 },
+        { days: 365, expectedSeconds: 31536000 }
+      ];
+
+      for (const { days, expectedSeconds } of testCases) {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: days })
+          .expect(200);
+
+        const indexes = await db.collection('user_activity').indexes();
+        const ttlIndex = indexes.find(
+          idx => idx.name === 'timestamp_ttl' || (idx.key && idx.key.timestamp === 1)
+        );
+
+        expect(ttlIndex).toBeDefined();
+        expect(ttlIndex.expireAfterSeconds).toBe(expectedSeconds);
+      }
+    });
+  });
+
+  // ============================================================
+  // SEC-4.1: Admin Action Audit Logging
+  // ============================================================
+  describe('SEC-4.1: Admin Action Audit Logging', () => {
+    describe('ADMIN_UPDATE_SETTINGS', () => {
+      it('should log ADMIN_UPDATE_SETTINGS action when updating settings', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+      });
+
+      it('should record admin user ID in audit log', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 120 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+      });
+
+      it('should record new retention value in audit log metadata', async () => {
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 150 })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.metadata).toHaveProperty('activityRetentionDays', 150);
+      });
+
+      it('should include timestamp in audit log', async () => {
+        const beforeTime = new Date();
+
+        await authAs(testUsers.admin)
+          .patch('/api/admin/settings')
+          .send({ activityRetentionDays: 100 })
+          .expect(200);
+
+        const afterTime = new Date();
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_UPDATE_SETTINGS'
+        });
+
+        expect(activity.timestamp).toBeDefined();
+        expect(new Date(activity.timestamp).getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+        expect(new Date(activity.timestamp).getTime()).toBeLessThanOrEqual(afterTime.getTime());
+      });
+    });
+
+    describe('ADMIN_DISABLE_USER', () => {
+      it('should log ADMIN_DISABLE_USER action when disabling a user', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}`)
+          .send({ isActive: false })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_DISABLE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+      });
+    });
+
+    describe('ADMIN_ENABLE_USER', () => {
+      it('should log ADMIN_ENABLE_USER action when enabling a user', async () => {
+        // First disable the user
+        await db.collection('users').updateOne(
+          { _id: testUsers.regularUser._id },
+          { $set: { isActive: false } }
+        );
+
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}`)
+          .send({ isActive: true })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_ENABLE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+      });
+    });
+
+    describe('ADMIN_PROMOTE_USER', () => {
+      it('should log ADMIN_PROMOTE_USER action when promoting to admin', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.regularUser._id}/role`)
+          .send({ role: 'admin' })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_PROMOTE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.regularUser._id.toString());
+        expect(activity.metadata.newRole).toBe('admin');
+      });
+    });
+
+    describe('ADMIN_DEMOTE_USER', () => {
+      beforeEach(async () => {
+        // Insert second admin for demotion tests
+        await db.collection('users').insertOne(testUsers.secondAdmin);
+      });
+
+      it('should log ADMIN_DEMOTE_USER action when demoting from admin', async () => {
+        await authAs(testUsers.admin)
+          .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+          .send({ role: 'user' })
+          .expect(200);
+
+        const activity = await db.collection('user_activity').findOne({
+          action: 'ADMIN_DEMOTE_USER'
+        });
+
+        expect(activity).toBeDefined();
+        expect(activity).not.toBeNull();
+        expect(activity.userId.toString()).toBe(testUsers.admin._id.toString());
+        expect(activity.metadata.targetUserId).toBe(testUsers.secondAdmin._id.toString());
+        expect(activity.metadata.newRole).toBe('user');
+      });
+    });
+  });
+
+  // ============================================================
+  // SEC-4.2: Last Admin Protection
+  // ============================================================
+  describe('SEC-4.2: Last Admin Protection', () => {
+    it('should return 400 when trying to demote the last admin', async () => {
+      // Ensure only one admin exists
+      await db.collection('users').deleteMany({
+        role: 'admin',
+        _id: { $ne: testUsers.admin._id }
+      });
+
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body).toHaveProperty('error');
+      expect(response.body.error).toMatch(/last admin|at least one admin|cannot demote/i);
+    });
+
+    it('should allow demoting an admin when other active admins exist', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Demote second admin (should succeed)
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+        .send({ role: 'user' })
+        .expect(200);
+
+      expect(response.body.data.user.role).toBe('user');
+    });
+
+    it('should not allow self-demotion even when other admins exist', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Try to demote self
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body).toHaveProperty('error');
+      expect(response.body.error).toMatch(/cannot demote yourself|self|own/i);
+    });
+
+    it('should count only active admins when checking for last admin', async () => {
+      // Insert a disabled admin (should not count)
+      const disabledAdmin = createTestAdmin({
+        _id: new ObjectId(),
+        email: 'disabled-admin@example.com',
+        isActive: false
+      });
+      await db.collection('users').insertOne(disabledAdmin);
+
+      // Try to demote self (should fail - only one active admin)
+      const response = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response.body.error).toMatch(/last admin|at least one admin|cannot demote/i);
+    });
+
+    it('should verify admin count after demotion would leave at least one admin', async () => {
+      // Insert second admin
+      await db.collection('users').insertOne(testUsers.secondAdmin);
+
+      // Verify we can demote the second admin
+      const response1 = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.secondAdmin._id}/role`)
+        .send({ role: 'user' })
+        .expect(200);
+
+      expect(response1.body.data.user.role).toBe('user');
+
+      // Now the primary admin is the last one, cannot demote
+      const response2 = await authAs(testUsers.admin)
+        .patch(`/api/admin/users/${testUsers.admin._id}/role`)
+        .send({ role: 'user' })
+        .expect(400);
+
+      expect(response2.body.error).toMatch(/last admin|at least one admin/i);
+    });
+  });
+
+  // ============================================================
+  // SEC-4.3: Rate Limiting on Admin Endpoints
+  // SKIPPED: Rate limiting moved to Phase 5
+  // ============================================================
+  describe.skip('SEC-4.3: Rate Limiting', () => {
+    /**
+     * Helper to make multiple rapid requests
+     * @param {string} endpoint - API endpoint
+     * @param {number} count - Number of requests
+     * @param {string} method - HTTP method
+     * @param {Object} body - Request body for POST/PATCH
+     * @returns {Promise<Array>} Array of responses
+     */
+    async function makeRapidRequests(endpoint, count, method = 'get', body = null) {
+      const requests = [];
+      for (let i = 0; i < count; i++) {
+        let req = authAs(testUsers.admin)[method](endpoint);
+        if (body && (method === 'patch' || method === 'post')) {
+          req = req.send(body);
+        }
+        requests.push(req);
+      }
+      return Promise.all(requests);
+    }
+
+    describe('User list endpoint rate limiting (60 req/min)', () => {
+      it('should allow up to 60 requests per minute to user list', async () => {
+        const responses = await makeRapidRequests('/api/admin/users', 60);
+
+        // All 60 requests should succeed
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(60);
+      });
+
+      it('should return 429 when exceeding 60 requests per minute', async () => {
+        const responses = await makeRapidRequests('/api/admin/users', 65);
+
+        // At least some should be rate limited
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+
+        // Rate limited response should have appropriate error
+        if (rateLimited.length > 0) {
+          expect(rateLimited[0].body).toHaveProperty('error');
+          expect(rateLimited[0].body.error).toMatch(/rate limit|too many requests/i);
+        }
+      });
+    });
+
+    describe('User updates endpoint rate limiting (30 req/min)', () => {
+      it('should allow up to 30 requests per minute for user updates', async () => {
+        const responses = await makeRapidRequests(
+          `/api/admin/users/${testUsers.regularUser._id}`,
+          30,
+          'patch',
+          { isActive: true }
+        );
+
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(30);
+      });
+
+      it('should return 429 when exceeding 30 user update requests per minute', async () => {
+        const responses = await makeRapidRequests(
+          `/api/admin/users/${testUsers.regularUser._id}`,
+          35,
+          'patch',
+          { isActive: true }
+        );
+
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe('Stats endpoints rate limiting (30 req/min)', () => {
+      const statsEndpoints = [
+        '/api/admin/stats/overview',
+        '/api/admin/stats/active-users',
+        '/api/admin/stats/actions',
+        '/api/admin/stats/peak-times'
+      ];
+
+      it('should allow up to 30 requests per minute to stats overview', async () => {
+        const responses = await makeRapidRequests('/api/admin/stats/overview', 30);
+        const successCount = responses.filter(r => r.status === 200).length;
+        expect(successCount).toBe(30);
+      });
+
+      it('should return 429 when exceeding 30 stats requests per minute', async () => {
+        const responses = await makeRapidRequests('/api/admin/stats/overview', 35);
+        const rateLimited = responses.filter(r => r.status === 429);
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+
+      it('should share rate limit across all stats endpoints', async () => {
+        // Make requests across different stats endpoints
+        const requests = [];
+        for (let i = 0; i < 40; i++) {
+          const endpoint = statsEndpoints[i % statsEndpoints.length];
+          requests.push(authAs(testUsers.admin).get(endpoint));
+        }
+
+        const responses = await Promise.all(requests);
+        const rateLimited = responses.filter(r => r.status === 429);
+
+        // After 30 total requests, should start rate limiting
+        expect(rateLimited.length).toBeGreaterThan(0);
+      });
+    });
+
+    describe('Rate limit headers', () => {
+      it('should include rate limit headers in response', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        // Check for common rate limit header variations
+        const hasRateLimitHeader =
+          response.headers['x-ratelimit-limit'] ||
+          response.headers['ratelimit-limit'] ||
+          response.headers['x-rate-limit-limit'];
+
+        expect(hasRateLimitHeader).toBeDefined();
+      });
+
+      it('should include remaining requests in headers', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        const hasRemainingHeader =
+          response.headers['x-ratelimit-remaining'] ||
+          response.headers['ratelimit-remaining'] ||
+          response.headers['x-rate-limit-remaining'];
+
+        expect(hasRemainingHeader).toBeDefined();
+      });
+
+      it('should include reset time in headers', async () => {
+        const response = await authAs(testUsers.admin)
+          .get('/api/admin/users')
+          .expect(200);
+
+        const hasResetHeader =
+          response.headers['x-ratelimit-reset'] ||
+          response.headers['ratelimit-reset'] ||
+          response.headers['x-rate-limit-reset'];
+
+        expect(hasResetHeader).toBeDefined();
+      });
+    });
+  });
+});
+
+// ============================================================
+// Integration Test: Full Settings Update Workflow
+// ============================================================
+describe('Integration: Settings Update Workflow', () => {
+  let app;
+  let db;
+  let client;
+  let testAdmin;
+
+  beforeAll(async () => {
+    const mongoUri = process.env.MONGO_URI || 'mongodb://localhost:27017';
+    client = new MongoClient(mongoUri);
+    await client.connect();
+    db = client.db('apartments_test');
+    app = await createTestApp(db);
+  });
+
+  afterAll(async () => {
+    if (db) {
+      await cleanupTestData(db);
+      await db.collection('settings').deleteMany({});
+    }
+    if (client) {
+      await client.close();
+    }
+  });
+
+  beforeEach(async () => {
+    await cleanupTestData(db);
+    await db.collection('settings').deleteMany({});
+
+    testAdmin = createTestAdmin({
+      _id: new ObjectId(),
+      email: 'integration-admin@example.com'
+    });
+    await db.collection('users').insertOne(testAdmin);
+
+    await db.collection('settings').insertOne({
+      _id: 'admin_settings',
+      activityRetentionDays: 90,
+      activityLogLevel: 'all',
+      updatedAt: new Date(),
+      updatedBy: null
+    });
+  });
+
+  it('should complete full settings update workflow', async () => {
+    const authAdmin = (method, url) => {
+      const token = generateTestToken(testAdmin._id);
+      return request(app)[method](url).set('Cookie', `auth_token=${token}`);
+    };
+
+    // Step 1: Get current settings
+    const getResponse = await authAdmin('get', '/api/admin/settings')
+      .expect(200);
+
+    expect(getResponse.body.activityRetentionDays).toBe(90);
+
+    // Step 2: Update settings to new value
+    const updateResponse = await authAdmin('patch', '/api/admin/settings')
+      .send({ activityRetentionDays: 180 })
+      .expect(200);
+
+    expect(updateResponse.body.data.settings.activityRetentionDays).toBe(180);
+    expect(updateResponse.body.data.message).toBe('Settings updated successfully');
+
+    // Step 3: Verify settings persisted
+    const verifyResponse = await authAdmin('get', '/api/admin/settings')
+      .expect(200);
+
+    expect(verifyResponse.body.activityRetentionDays).toBe(180);
+
+    // Step 4: Verify audit log was created
+    const auditLog = await db.collection('user_activity').findOne({
+      action: 'ADMIN_UPDATE_SETTINGS'
+    });
+
+    expect(auditLog).toBeDefined();
+    expect(auditLog).not.toBeNull();
+    expect(auditLog.metadata.activityRetentionDays).toBe(180);
+    expect(auditLog.userId.toString()).toBe(testAdmin._id.toString());
+
+    // Step 5: Verify TTL index was updated
+    const indexes = await db.collection('user_activity').indexes();
+    const ttlIndex = indexes.find(idx => idx.name === 'timestamp_ttl');
+
+    expect(ttlIndex).toBeDefined();
+    expect(ttlIndex.expireAfterSeconds).toBe(180 * 24 * 60 * 60);
+
+    // Step 6: Verify database document was updated
+    const dbSettings = await db.collection('settings').findOne({ _id: 'admin_settings' });
+    expect(dbSettings.activityRetentionDays).toBe(180);
+    expect(dbSettings.updatedBy.toString()).toBe(testAdmin._id.toString());
+  });
+
+  it('should handle multiple sequential settings updates', async () => {
+    const authAdmin = (method, url) => {
+      const token = generateTestToken(testAdmin._id);
+      return request(app)[method](url).set('Cookie', `auth_token=${token}`);
+    };
+
+    const retentionValues = [30, 60, 90, 180, 365];
+
+    for (const value of retentionValues) {
+      const response = await authAdmin('patch', '/api/admin/settings')
+        .send({ activityRetentionDays: value })
+        .expect(200);
+
+      expect(response.body.data.settings.activityRetentionDays).toBe(value);
+
+      // Verify TTL index after each update
+      const indexes = await db.collection('user_activity').indexes();
+      const ttlIndex = indexes.find(idx => idx.name === 'timestamp_ttl');
+      expect(ttlIndex.expireAfterSeconds).toBe(value * 24 * 60 * 60);
+    }
+
+    // Should have 5 audit log entries
+    const auditLogCount = await db.collection('user_activity').countDocuments({
+      action: 'ADMIN_UPDATE_SETTINGS'
+    });
+    expect(auditLogCount).toBe(5);
+  });
+});

__tests__/scraper/dataConversion.test.js

@@ -0,0 +1,496 @@
+/**
+ * Tests for convertDataTypes() and its internal helper functions
+ *
+ * Covers:
+ * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
+ * - parseInteger: valid integers, null/empty/invalid
+ * - parsePositiveInteger: treats 0 as null (for area)
+ * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
+ * - parseFloat: "1.5" for half baths, null/empty/invalid
+ * - parseBoolean: "true"/"false"/"1"/"0", null/empty
+ * - trimString: trim whitespace, null for empty, null/undefined
+ * - convertDataTypes: full integration applying correct parser to each field
+ */
+
+const {
+  convertDataTypes,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString
+} = require('../../services/scraperService');
+
+// ---------------------------------------------------------------
+// 1. parsePrice
+// ---------------------------------------------------------------
+describe('parsePrice', () => {
+  it('should parse "$1,234" format to 1234', () => {
+    expect(parsePrice('$1,234')).toBe(1234);
+  });
+
+  it('should parse plain numeric string', () => {
+    expect(parsePrice('1450')).toBe(1450);
+  });
+
+  it('should parse "$2,500.00" stripping non-numeric chars', () => {
+    expect(parsePrice('$2,500.00')).toBe(2500);
+  });
+
+  it('should return null for "Call for pricing"', () => {
+    expect(parsePrice('Call for pricing')).toBeNull();
+  });
+
+  it('should return null for "Contact us"', () => {
+    expect(parsePrice('Contact us')).toBeNull();
+  });
+
+  it('should return null for "Please inquire"', () => {
+    expect(parsePrice('Please inquire')).toBeNull();
+  });
+
+  it('should return null for negative values', () => {
+    // After stripping non-numeric chars, "-1500" becomes "1500"
+    // but if someone passes a numeric -1500 directly, check that too
+    expect(parsePrice('-')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(parsePrice(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePrice('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parsePrice(undefined)).toBeNull();
+  });
+
+  it('should return null for completely non-numeric string', () => {
+    expect(parsePrice('abc')).toBeNull();
+  });
+
+  it('should handle price of 0', () => {
+    expect(parsePrice('0')).toBe(0);
+  });
+
+  it('should handle large prices', () => {
+    expect(parsePrice('$15,500')).toBe(15500);
+  });
+});
+
+// ---------------------------------------------------------------
+// 2. parseInteger
+// ---------------------------------------------------------------
+describe('parseInteger', () => {
+  it('should parse valid integer string', () => {
+    expect(parseInteger('42')).toBe(42);
+  });
+
+  it('should parse "0" to 0', () => {
+    expect(parseInteger('0')).toBe(0);
+  });
+
+  it('should parse negative integer string', () => {
+    expect(parseInteger('-5')).toBe(-5);
+  });
+
+  it('should return null for null', () => {
+    expect(parseInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseInteger('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseInteger(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseInteger('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseInteger('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseInteger('Infinity')).toBeNull();
+  });
+
+  it('should truncate float strings to integer', () => {
+    expect(parseInteger('3.7')).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------
+// 3. parsePositiveInteger
+// ---------------------------------------------------------------
+describe('parsePositiveInteger', () => {
+  it('should parse positive integer normally', () => {
+    expect(parsePositiveInteger('750')).toBe(750);
+  });
+
+  it('should treat 0 as null (for area field)', () => {
+    expect(parsePositiveInteger('0')).toBeNull();
+  });
+
+  it('should return null for null input', () => {
+    expect(parsePositiveInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePositiveInteger('')).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parsePositiveInteger('abc')).toBeNull();
+  });
+
+  it('should handle negative values (parsed as negative int, not positive)', () => {
+    expect(parsePositiveInteger('-5')).toBe(-5);
+  });
+});
+
+// ---------------------------------------------------------------
+// 4. parseIntegerOrString
+// ---------------------------------------------------------------
+describe('parseIntegerOrString', () => {
+  it('should return integer when value is cleanly parseable', () => {
+    expect(parseIntegerOrString('5001')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric value', () => {
+    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
+  });
+
+  it('should return trimmed string when value has non-numeric suffix', () => {
+    expect(parseIntegerOrString('123abc')).toBe('123abc');
+  });
+
+  it('should return null for null', () => {
+    expect(parseIntegerOrString(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseIntegerOrString('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseIntegerOrString(undefined)).toBeNull();
+  });
+
+  it('should trim whitespace before checking', () => {
+    expect(parseIntegerOrString('  5001  ')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric with whitespace', () => {
+    expect(parseIntegerOrString('  abc  ')).toBe('abc');
+  });
+});
+
+// ---------------------------------------------------------------
+// 5. parseFloatValue
+// ---------------------------------------------------------------
+describe('parseFloatValue', () => {
+  it('should parse "1.5" for half baths', () => {
+    expect(parseFloatValue('1.5')).toBe(1.5);
+  });
+
+  it('should parse "2.0" to 2', () => {
+    expect(parseFloatValue('2.0')).toBe(2.0);
+  });
+
+  it('should parse integer string as float', () => {
+    expect(parseFloatValue('3')).toBe(3);
+  });
+
+  it('should return null for null', () => {
+    expect(parseFloatValue(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseFloatValue('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseFloatValue(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseFloatValue('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseFloatValue('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseFloatValue('Infinity')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 6. parseBoolean
+// ---------------------------------------------------------------
+describe('parseBoolean', () => {
+  it('should parse "true" to true', () => {
+    expect(parseBoolean('true')).toBe(true);
+  });
+
+  it('should parse "false" to false', () => {
+    expect(parseBoolean('false')).toBe(false);
+  });
+
+  it('should parse "1" to true', () => {
+    expect(parseBoolean('1')).toBe(true);
+  });
+
+  it('should parse "0" to false', () => {
+    expect(parseBoolean('0')).toBe(false);
+  });
+
+  it('should parse "TRUE" (case insensitive) to true', () => {
+    expect(parseBoolean('TRUE')).toBe(true);
+  });
+
+  it('should parse "False" (case insensitive) to false', () => {
+    expect(parseBoolean('False')).toBe(false);
+  });
+
+  it('should pass through boolean true', () => {
+    expect(parseBoolean(true)).toBe(true);
+  });
+
+  it('should pass through boolean false', () => {
+    expect(parseBoolean(false)).toBe(false);
+  });
+
+  it('should return null for null', () => {
+    expect(parseBoolean(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseBoolean('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseBoolean(undefined)).toBeNull();
+  });
+
+  it('should return null for unrecognized string', () => {
+    expect(parseBoolean('yes')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 7. trimString
+// ---------------------------------------------------------------
+describe('trimString', () => {
+  it('should trim leading and trailing whitespace', () => {
+    expect(trimString('  hello  ')).toBe('hello');
+  });
+
+  it('should return the string as-is if no whitespace', () => {
+    expect(trimString('hello')).toBe('hello');
+  });
+
+  it('should return null for empty string', () => {
+    expect(trimString('')).toBeNull();
+  });
+
+  it('should return null for whitespace-only string', () => {
+    expect(trimString('   ')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(trimString(null)).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(trimString(undefined)).toBeNull();
+  });
+
+  it('should convert non-string to string then trim', () => {
+    expect(trimString(123)).toBe('123');
+  });
+});
+
+// ---------------------------------------------------------------
+// 8. convertDataTypes - Full integration test
+// ---------------------------------------------------------------
+describe('convertDataTypes', () => {
+  it('should apply correct parser to each field', () => {
+    const rawUnit = {
+      id: '1001',
+      unit_code: 'CCT-101',
+      unit_id: '5001',
+      floor: '1',
+      area: '750',
+      bed_count: '1',
+      bath_count: '1.5',
+      price: '1450',
+      available: 'true',
+      unavailable: 'false',
+      soonest: '2026-03-01',
+      date_available: '20260301',
+      plan_id: '201',
+      plan_name: 'Alpine',
+      obj_type: 'unit',
+      community: 'Country Club Towers',
+      asset: '100',
+      href: '/units/CCT-101',
+      inventory_href: '/inventory/CCT-101',
+      image_url: 'https://example.com/images/unit-101.jpg',
+      specials_content: 'First month free!'
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    // Integer fields
+    expect(converted.id).toBe(1001);
+    expect(converted.floor).toBe(1);
+    expect(converted.bed_count).toBe(1);
+    expect(converted.plan_id).toBe(201);
+    expect(converted.asset).toBe(100);
+    expect(converted.date_available).toBe(20260301);
+
+    // parseIntegerOrString
+    expect(converted.unit_id).toBe(5001);
+
+    // parsePositiveInteger (area)
+    expect(converted.area).toBe(750);
+
+    // Float field (bath_count)
+    expect(converted.bath_count).toBe(1.5);
+
+    // Price field
+    expect(converted.price).toBe(1450);
+
+    // Boolean fields
+    expect(converted.available).toBe(true);
+    expect(converted.unavailable).toBe(false);
+
+    // String fields (trimmed)
+    expect(converted.unit_code).toBe('CCT-101');
+    expect(converted.plan_name).toBe('Alpine');
+    expect(converted.soonest).toBe('2026-03-01');
+    expect(converted.obj_type).toBe('unit');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.href).toBe('/units/CCT-101');
+    expect(converted.inventory_href).toBe('/inventory/CCT-101');
+    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
+    expect(converted.specials_content).toBe('First month free!');
+  });
+
+  it('should preserve string fields without converting them', () => {
+    const rawUnit = {
+      unit_code: 'CCT-205',
+      plan_name: '  Birch  ',
+      soonest: 'Now',
+      obj_type: 'unit',
+      community: '  Country Club Towers  ',
+      href: '/units/CCT-205',
+      inventory_href: '/inventory/CCT-205',
+      image_url: 'https://example.com/img.jpg',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.unit_code).toBe('CCT-205');
+    expect(converted.plan_name).toBe('Birch');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle a unit with all null/undefined/empty values', () => {
+    const rawUnit = {
+      id: null,
+      unit_code: undefined,
+      unit_id: '',
+      floor: null,
+      area: '',
+      bed_count: undefined,
+      bath_count: null,
+      price: '',
+      available: null,
+      unavailable: undefined,
+      soonest: '',
+      date_available: null,
+      plan_id: undefined,
+      plan_name: '',
+      obj_type: null,
+      community: undefined,
+      asset: '',
+      href: null,
+      inventory_href: undefined,
+      image_url: '',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.id).toBeNull();
+    expect(converted.unit_code).toBeNull();
+    expect(converted.unit_id).toBeNull();
+    expect(converted.floor).toBeNull();
+    expect(converted.area).toBeNull();
+    expect(converted.bed_count).toBeNull();
+    expect(converted.bath_count).toBeNull();
+    expect(converted.price).toBeNull();
+    expect(converted.available).toBeNull();
+    expect(converted.unavailable).toBeNull();
+    expect(converted.soonest).toBeNull();
+    expect(converted.date_available).toBeNull();
+    expect(converted.plan_id).toBeNull();
+    expect(converted.plan_name).toBeNull();
+    expect(converted.obj_type).toBeNull();
+    expect(converted.community).toBeNull();
+    expect(converted.asset).toBeNull();
+    expect(converted.href).toBeNull();
+    expect(converted.inventory_href).toBeNull();
+    expect(converted.image_url).toBeNull();
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle area=0 as null (parsePositiveInteger)', () => {
+    const rawUnit = { area: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.area).toBeNull();
+  });
+
+  it('should handle bed_count=0 as valid (studio)', () => {
+    const rawUnit = { bed_count: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bed_count).toBe(0);
+  });
+
+  it('should handle "Call for pricing" in price field', () => {
+    const rawUnit = { price: 'Call for pricing' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBeNull();
+  });
+
+  it('should handle "$1,234" price format', () => {
+    const rawUnit = { price: '$1,234' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBe(1234);
+  });
+
+  it('should handle unit_id as string identifier', () => {
+    const rawUnit = { unit_id: 'UNIT-ABC' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.unit_id).toBe('UNIT-ABC');
+  });
+
+  it('should handle bath_count with half baths', () => {
+    const rawUnit = { bath_count: '1.5' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bath_count).toBe(1.5);
+  });
+});

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

__tests__/scraper/fixtures/sample-listing.html

@@ -0,0 +1,144 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Country Club Towers - Available Units</title>
+</head>
+<body>
+  <div class="spaces-container">
+    <section class="spaces__tab-unit">
+      <!-- Unit 1: Fully populated with all attributes and an image -->
+      <article
+        data-spaces-id="1001"
+        data-spaces-unit="CCT-101"
+        data-spaces-unit-id="5001"
+        data-spaces-unit-floor="1"
+        data-spaces-sort-area="750"
+        data-spaces-sort-bed="1"
+        data-spaces-sort-bath="1"
+        data-spaces-sort-price="1450"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="2026-03-01"
+        data-spaces-sort-date="20260301"
+        data-spaces-plan-id="201"
+        data-spaces-sort-plan-name="Alpine"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-101"
+        data-spaces-inventory-href="/inventory/CCT-101"
+        data-spaces-specials-content="First month free!"
+      >
+        <div class="unit-card">
+          <img src="https://example.com/images/unit-101.jpg" alt="Unit 101" />
+          <h3>Unit CCT-101</h3>
+          <p>1 Bed / 1 Bath - $1,450/mo</p>
+        </div>
+      </article>
+
+      <!-- Unit 2: Different floor plan, no specials, with data-src image -->
+      <article
+        data-spaces-id="1002"
+        data-spaces-unit="CCT-205"
+        data-spaces-unit-id="5002"
+        data-spaces-unit-floor="2"
+        data-spaces-sort-area="950"
+        data-spaces-sort-bed="2"
+        data-spaces-sort-bath="1.5"
+        data-spaces-sort-price="1850"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="2026-02-15"
+        data-spaces-sort-date="20260215"
+        data-spaces-plan-id="202"
+        data-spaces-sort-plan-name="Birchwood"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-205"
+        data-spaces-inventory-href="/inventory/CCT-205"
+        data-spaces-specials-content=""
+      >
+        <div class="unit-card">
+          <img data-src="https://example.com/images/unit-205.jpg" alt="Unit 205" />
+          <h3>Unit CCT-205</h3>
+        </div>
+      </article>
+
+      <!-- Unit 3: Studio with some missing attributes -->
+      <article
+        data-spaces-id="1003"
+        data-spaces-unit="CCT-310"
+        data-spaces-unit-id="5003"
+        data-spaces-unit-floor="3"
+        data-spaces-sort-area="500"
+        data-spaces-sort-bed="0"
+        data-spaces-sort-bath="1"
+        data-spaces-sort-price="1100"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="Available Now"
+        data-spaces-sort-date="20260201"
+        data-spaces-plan-id="203"
+        data-spaces-sort-plan-name="Cedar"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-310"
+        data-spaces-inventory-href="/inventory/CCT-310"
+      >
+        <div class="unit-card">
+          <h3>Unit CCT-310</h3>
+          <p>Studio - $1,100/mo</p>
+        </div>
+      </article>
+
+      <!-- Unit 4: Unavailable unit -->
+      <article
+        data-spaces-id="1004"
+        data-spaces-unit="CCT-415"
+        data-spaces-unit-id="5004"
+        data-spaces-unit-floor="4"
+        data-spaces-sort-area="1200"
+        data-spaces-sort-bed="3"
+        data-spaces-sort-bath="2"
+        data-spaces-sort-price="2500"
+        data-spaces-available="false"
+        data-spaces-unavailable="true"
+        data-spaces-soonest=""
+        data-spaces-sort-date=""
+        data-spaces-plan-id="204"
+        data-spaces-sort-plan-name="Dogwood"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-415"
+        data-spaces-inventory-href="/inventory/CCT-415"
+        data-spaces-specials-content=""
+      >
+        <div class="unit-card">
+          <img src="https://example.com/images/unit-415.jpg" alt="Unit 415" />
+          <h3>Unit CCT-415</h3>
+        </div>
+      </article>
+
+      <!-- Unit 5: Minimal attributes - edge case -->
+      <article
+        data-spaces-id="1005"
+        data-spaces-unit="CCT-520"
+        data-spaces-unit-id="5005"
+        data-spaces-sort-price="1300"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+      >
+        <div class="unit-card">
+          <h3>Unit CCT-520</h3>
+        </div>
+      </article>
+    </section>
+  </div>
+</body>
+</html>

__tests__/scraper/insertPrices.test.js

@@ -0,0 +1,582 @@
+/**
+ * Tests for insertPrices() bulk operation
+ *
+ * Covers:
+ * - Only units with non-null prices are inserted
+ * - bulkWrite uses updateOne with upsert for idempotency
+ * - Filter uses unit_code + date_checked composite key
+ * - Price record includes unit_code, date_checked, price, last_updated, data_source
+ * - Empty price records logs warning and returns early
+ * - Error handling when bulkWrite fails
+ * - Logging of inserted and updated counts
+ * - Re-running on same day updates existing records (idempotent)
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require insertPrices after implementation
+let insertPrices;
+
+let mongoServer;
+let client;
+let db;
+
+const PRICES_COLLECTION = 'unit_prices_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ insertPrices } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the prices collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('insertPrices', () => {
+  const TODAY = '2026-02-05';
+
+  // ---------------------------------------------------------------
+  // 1. Only units with non-null prices are inserted
+  // ---------------------------------------------------------------
+  describe('filtering units with null prices', () => {
+    it('should only insert records for units with non-null prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200 },
+        { unit_code: 'A102', price: null },
+        { unit_code: 'A103', price: 1500 },
+        { unit_code: 'A104', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'A103']);
+    });
+
+    it('should skip units with price of null and not create any record for them', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'B101', price: null },
+        { unit_code: 'B102', price: null }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(0);
+    });
+
+    it('should handle a mix of priced and null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'C101', price: 950 },
+        { unit_code: 'C102', price: null },
+        { unit_code: 'C103', price: 1100 },
+        { unit_code: 'C104', price: null },
+        { unit_code: 'C105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. bulkWrite uses updateOne with upsert for idempotency
+  // ---------------------------------------------------------------
+  describe('bulkWrite with upsert operations', () => {
+    it('should create updateOne operations for each priced unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'D101', price: 1200 },
+        { unit_code: 'D102', price: 1500 }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.insertedCount).toBeGreaterThan(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+    });
+
+    it('should use upsert so re-running does not create duplicates', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'D201', price: 1200 }];
+
+      // First run
+      await insertPrices(db, units, TODAY, logger);
+
+      // Second run on same day
+      await insertPrices(db, units, TODAY, logger);
+
+      // Should still have only 1 document (not 2)
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Filter uses unit_code + date_checked composite key
+  // ---------------------------------------------------------------
+  describe('composite key: unit_code + date_checked', () => {
+    it('should use unit_code and date_checked as the filter for upsert', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E101', price: 1200 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'E101',
+        date_checked: TODAY
+      });
+
+      expect(doc).not.toBeNull();
+      expect(doc.unit_code).toBe('E101');
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should create separate records for the same unit on different days', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E201', price: 1200 }];
+
+      await insertPrices(db, units, '2026-02-04', logger);
+      await insertPrices(db, units, '2026-02-05', logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ unit_code: 'E201' })
+        .sort({ date_checked: 1 })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+      expect(docs[0].date_checked).toBe('2026-02-04');
+      expect(docs[1].date_checked).toBe('2026-02-05');
+    });
+
+    it('should create separate records for different units on the same day', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'E301', price: 1000 },
+        { unit_code: 'E302', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ date_checked: TODAY })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Price record includes required fields
+  // ---------------------------------------------------------------
+  describe('price record fields', () => {
+    it('should include unit_code in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.unit_code).toBe('F101');
+    });
+
+    it('should include date_checked in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F201', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should include price as a number in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F301', price: 1550 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.price).toBe(1550);
+      expect(typeof doc.price).toBe('number');
+    });
+
+    it('should include last_updated as an ISO timestamp string', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'F401', price: 1600 }];
+      await insertPrices(db, units, TODAY, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.last_updated).toBeDefined();
+      expect(typeof doc.last_updated).toBe('string');
+      expect(doc.last_updated >= beforeTime).toBe(true);
+      expect(doc.last_updated <= afterTime).toBe(true);
+    });
+
+    it('should include data_source set to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F501', price: 1700 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should include all five required fields in every price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F601', price: 1800 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc).toHaveProperty('unit_code');
+      expect(doc).toHaveProperty('date_checked');
+      expect(doc).toHaveProperty('price');
+      expect(doc).toHaveProperty('last_updated');
+      expect(doc).toHaveProperty('data_source');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty price records logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty price records', () => {
+    it('should log a warning when no units have prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'G101', price: null },
+        { unit_code: 'G102', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should return insertedCount of 0 for all null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'G201', price: null }];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+    });
+
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should not perform any database writes for empty/null-priced arrays', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H101', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H201', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await insertPrices(mockDb, units, TODAY, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to insert prices',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H301', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Logging of inserted and updated counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log inserted and updated counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I101', price: 1000 },
+        { unit_code: 'I102', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: expect.any(Number),
+          updated: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I201', price: 1000 },
+        { unit_code: 'I202', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // For new inserts: inserted=2, updated=0
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 2,
+          updated: 0
+        })
+      );
+    });
+
+    it('should log correct counts when updating existing records', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'I301', price: 1000 }];
+
+      // First insert
+      await insertPrices(db, units, TODAY, logger);
+
+      // Reset mock
+      logger.info.mockClear();
+
+      // Second run updates existing
+      const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
+      await insertPrices(db, updatedUnits, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 0,
+          updated: 1
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Re-running on same day updates existing records (idempotent)
+  // ---------------------------------------------------------------
+  describe('idempotent re-runs on same day', () => {
+    it('should update the price when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run with original price
+      await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
+
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc1.price).toBe(1200);
+
+      // Re-run with updated price (same day)
+      await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
+
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc2.price).toBe(1250);
+
+      // Should still have only 1 record
+      const count = await db.collection(PRICES_COLLECTION).countDocuments({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(count).toBe(1);
+    });
+
+    it('should update last_updated when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run
+      await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const firstUpdated = doc1.last_updated;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Re-run on same day
+      await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const secondUpdated = doc2.last_updated;
+
+      expect(secondUpdated > firstUpdated).toBe(true);
+    });
+
+    it('should return combined insert/update count reflecting the operation', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'J301', price: 1000 },
+        { unit_code: 'J302', price: 1100 }
+      ];
+
+      // First run - inserts
+      const result1 = await insertPrices(db, units, TODAY, logger);
+      expect(result1.insertedCount).toBe(2);
+
+      // Re-run - updates
+      const result2 = await insertPrices(db, units, TODAY, logger);
+      expect(result2.insertedCount).toBe(2); // combined upserted + modified
+
+      // Still only 2 total records
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(2);
+    });
+
+    it('should handle a mix of new and existing records on re-run', async () => {
+      const logger = createMockLogger();
+
+      // First run with unit A
+      await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
+
+      // Re-run with unit A (existing) and unit B (new)
+      const mixedUnits = [
+        { unit_code: 'J401', price: 1050 },
+        { unit_code: 'J402', price: 1200 }
+      ];
+      await insertPrices(db, mixedUnits, TODAY, logger);
+
+      // Should have 2 total records
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      // Verify updated price
+      const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
+      expect(unitA.price).toBe(1050);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Uses ordered: false for performance
+  // ---------------------------------------------------------------
+  describe('ordered: false for performance', () => {
+    it('should successfully process all priced units in a batch', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'K101', price: 900 },
+        { unit_code: 'K102', price: 1000 },
+        { unit_code: 'K103', price: 1100 },
+        { unit_code: 'K104', price: 1200 },
+        { unit_code: 'K105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Uses correct collection
+  // ---------------------------------------------------------------
+  describe('collection usage', () => {
+    it('should write to the configured PRICES collection', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'L101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // Verify data is in the correct collection
+      const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
+      expect(doc).not.toBeNull();
+    });
+  });
+});

__tests__/scraper/markStaleUnits.test.js

@@ -0,0 +1,449 @@
+/**
+ * Tests for markStaleUnits() availability tracking
+ *
+ * Covers:
+ * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
+ * - $set updates available to false and sets marked_stale_date
+ * - Units in currentCodes are NOT marked stale
+ * - Already unavailable units are not modified
+ * - Error handling when updateMany fails
+ * - Logging of modified count
+ * - Empty currentUnitCodes handles gracefully
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require markStaleUnits after implementation
+let markStaleUnits;
+
+let mongoServer;
+let client;
+let db;
+
+const UNITS_COLLECTION = 'units_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ markStaleUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('markStaleUnits', () => {
+  // ---------------------------------------------------------------
+  // 1. updateMany is called with correct filter
+  //    (unit_code $nin currentCodes AND available: true)
+  // ---------------------------------------------------------------
+  describe('filter criteria', () => {
+    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
+      const logger = createMockLogger();
+
+      // Seed units in the database
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'A101', available: true, price: 1200 },
+        { unit_code: 'A102', available: true, price: 1300 },
+        { unit_code: 'B201', available: true, price: 1500 }
+      ]);
+
+      // Current scrape found A101 and A102, but NOT B201
+      const currentCodes = new Set(['A101', 'A102']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // B201 should be marked stale (it was available but not in current scrape)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify B201 was updated
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
+      expect(staleUnit.available).toBe(false);
+      expect(staleUnit.marked_stale_date).toBe(date);
+    });
+
+    it('should only affect units NOT in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'X1', available: true },
+        { unit_code: 'X2', available: true },
+        { unit_code: 'X3', available: true },
+        { unit_code: 'X4', available: true }
+      ]);
+
+      // Current scrape found X1 and X3
+      const currentCodes = new Set(['X1', 'X3']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // X1 and X3 should still be available
+      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
+      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
+      expect(x1.available).toBe(true);
+      expect(x3.available).toBe(true);
+
+      // X2 and X4 should be marked stale
+      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
+      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
+      expect(x2.available).toBe(false);
+      expect(x4.available).toBe(false);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. $set updates available to false and sets marked_stale_date
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set available to false for stale units', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'S100', available: true, price: 1000 },
+        { unit_code: 'S200', available: true, price: 1100 }
+      ]);
+
+      const currentCodes = new Set(['S100']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
+      expect(staleUnit.available).toBe(false);
+    });
+
+    it('should set marked_stale_date to the provided date', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'D100', available: true }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
+      expect(doc.marked_stale_date).toBe('2026-02-05');
+    });
+
+    it('should update both available and marked_stale_date in the same operation', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'C100', available: true, price: 900 }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-01-15';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
+      expect(doc.available).toBe(false);
+      expect(doc.marked_stale_date).toBe('2026-01-15');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Units in currentCodes are NOT marked stale
+  // ---------------------------------------------------------------
+  describe('units in currentCodes not affected', () => {
+    it('should not modify units that are in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'KEEP-1', available: true, price: 1200 },
+        { unit_code: 'KEEP-2', available: true, price: 1300 },
+        { unit_code: 'STALE-1', available: true, price: 1400 }
+      ]);
+
+      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // Kept units should retain their original state
+      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
+      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
+
+      expect(keep1.available).toBe(true);
+      expect(keep1.marked_stale_date).toBeUndefined();
+      expect(keep2.available).toBe(true);
+      expect(keep2.marked_stale_date).toBeUndefined();
+    });
+
+    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ALL-1', available: true },
+        { unit_code: 'ALL-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['ALL-1', 'ALL-2']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Already unavailable units are not modified
+  // ---------------------------------------------------------------
+  describe('already unavailable units', () => {
+    it('should not modify units that are already unavailable (available: false)', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
+        { unit_code: 'UNAVAIL-2', available: false },
+        { unit_code: 'AVAIL-1', available: true }
+      ]);
+
+      // None of these units are in the current scrape
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify the already-unavailable unit's date was NOT changed
+      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
+      expect(unavail1.marked_stale_date).toBe('2026-01-01');
+    });
+
+    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'U1', available: false },
+        { unit_code: 'U2', available: false }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Error handling when updateMany fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      try {
+        await markStaleUnits(mockDb, currentCodes, date, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to mark stale units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging of modified count
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of stale units marked', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'LOG-1', available: true },
+        { unit_code: 'LOG-2', available: true },
+        { unit_code: 'LOG-3', available: true }
+      ]);
+
+      const currentCodes = new Set(['LOG-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 2 })
+      );
+    });
+
+    it('should log count of 0 when no units are marked stale', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ZERO-1', available: true }
+      ]);
+
+      const currentCodes = new Set(['ZERO-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Empty currentUnitCodes handles gracefully
+  // ---------------------------------------------------------------
+  describe('empty currentUnitCodes', () => {
+    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'EMPTY-1', available: true },
+        { unit_code: 'EMPTY-2', available: true },
+        { unit_code: 'EMPTY-3', available: true }
+      ]);
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(3);
+
+      // All should be marked unavailable
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      for (const doc of docs) {
+        expect(doc.available).toBe(false);
+        expect(doc.marked_stale_date).toBe('2026-02-05');
+      }
+    });
+
+    it('should not throw when currentUnitCodes is an empty Set', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      // Should not throw even with empty collection and empty set
+      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
+    });
+
+    it('should handle empty database with empty currentUnitCodes', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the MongoDB updateMany result', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'RET-1', available: true },
+        { unit_code: 'RET-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['RET-1']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // updateMany result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('acknowledged');
+      expect(result.acknowledged).toBe(true);
+    });
+  });
+});

__tests__/scraper/parseUnits.test.js

@@ -0,0 +1,360 @@
+/**
+ * Tests for parseUnits() - HTML parsing with cheerio
+ *
+ * Covers:
+ * - Extraction of all data-spaces-* attributes
+ * - Image URL extraction from nested img elements
+ * - Container detection (section.spaces__tab-unit)
+ * - Empty / missing container handling
+ * - Deduplication by unit_code
+ * - Logging behavior (info, warn, error)
+ * - Missing attribute handling
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseUnits } = require('../../services/scraperService');
+
+const fixturesDir = path.join(__dirname, 'fixtures');
+
+// Helper: create a mock logger
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+describe('parseUnits', () => {
+  let sampleHtml;
+  let mockLogger;
+
+  beforeAll(() => {
+    sampleHtml = fs.readFileSync(
+      path.join(fixturesDir, 'sample-listing.html'),
+      'utf-8'
+    );
+  });
+
+  beforeEach(() => {
+    mockLogger = createMockLogger();
+  });
+
+  // ---------------------------------------------------------------
+  // 1. Container detection
+  // ---------------------------------------------------------------
+  describe('container detection', () => {
+    it('should find section.spaces__tab-unit container in valid HTML', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+
+      // Should not log an error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+      expect(units.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array when section container is missing', () => {
+      const html = '<html><body><div>No units here</div></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+
+    it('should return empty array for empty HTML string', () => {
+      const units = parseUnits('', mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    it('should return empty array when container exists but has no articles', () => {
+      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container found, no error about container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Extraction of all data-spaces-* attributes
+  // ---------------------------------------------------------------
+  describe('attribute extraction', () => {
+    it('should extract all expected attributes from a fully populated article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
+
+      expect(unit1).toBeDefined();
+
+      // Core identifiers
+      expect(unit1.id).toBe('1001');
+      expect(unit1.unit_code).toBe('CCT-101');
+      expect(unit1.unit_id).toBe('5001');
+
+      // Physical attributes
+      expect(unit1.floor).toBe('1');
+      expect(unit1.area).toBe('750');
+      expect(unit1.bed_count).toBe('1');
+      expect(unit1.bath_count).toBe('1');
+
+      // Pricing
+      expect(unit1.price).toBe('1450');
+
+      // Availability
+      expect(unit1.available).toBe('true');
+      expect(unit1.unavailable).toBe('false');
+      expect(unit1.soonest).toBe('2026-03-01');
+      expect(unit1.date_available).toBe('20260301');
+
+      // Plan information
+      expect(unit1.plan_id).toBe('201');
+      expect(unit1.plan_name).toBe('Alpine');
+
+      // Property information
+      expect(unit1.obj_type).toBe('unit');
+      expect(unit1.community).toBe('Country Club Towers');
+      expect(unit1.asset).toBe('100');
+
+      // URLs
+      expect(unit1.href).toBe('/units/CCT-101');
+      expect(unit1.inventory_href).toBe('/inventory/CCT-101');
+
+      // Specials
+      expect(unit1.specials_content).toBe('First month free!');
+    });
+
+    it('should extract correct count of units from the fixture', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      expect(units).toHaveLength(5);
+    });
+
+    it('should return raw string values without type conversion', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'CCT-101');
+
+      // All values should be strings (raw extraction, no conversion)
+      expect(typeof unit.id).toBe('string');
+      expect(typeof unit.price).toBe('string');
+      expect(typeof unit.floor).toBe('string');
+      expect(typeof unit.available).toBe('string');
+      expect(typeof unit.bed_count).toBe('string');
+      expect(typeof unit.bath_count).toBe('string');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Image URL extraction
+  // ---------------------------------------------------------------
+  describe('image URL extraction', () => {
+    it('should extract image_url from img src attribute', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
+
+      expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
+    });
+
+    it('should fall back to data-src when src is not present', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
+
+      expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
+    });
+
+    it('should not set image_url when no img element exists', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
+
+      expect(unit3.image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Missing attributes
+  // ---------------------------------------------------------------
+  describe('missing attributes', () => {
+    it('should return undefined for attributes not present on the article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
+      // soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
+
+      expect(unit5).toBeDefined();
+      expect(unit5.floor).toBeUndefined();
+      expect(unit5.area).toBeUndefined();
+      expect(unit5.bed_count).toBeUndefined();
+      expect(unit5.bath_count).toBeUndefined();
+      expect(unit5.soonest).toBeUndefined();
+      expect(unit5.date_available).toBeUndefined();
+      expect(unit5.plan_id).toBeUndefined();
+      expect(unit5.plan_name).toBeUndefined();
+      expect(unit5.href).toBeUndefined();
+      expect(unit5.inventory_href).toBeUndefined();
+      expect(unit5.specials_content).toBeUndefined();
+    });
+
+    it('should still extract the attributes that are present on a sparse article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
+
+      expect(unit5.id).toBe('1005');
+      expect(unit5.unit_code).toBe('CCT-520');
+      expect(unit5.unit_id).toBe('5005');
+      expect(unit5.price).toBe('1300');
+      expect(unit5.available).toBe('true');
+      expect(unit5.obj_type).toBe('unit');
+      expect(unit5.community).toBe('Country Club Towers');
+      expect(unit5.asset).toBe('100');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Deduplication
+  // ---------------------------------------------------------------
+  describe('deduplication', () => {
+    it('should deduplicate units with the same unit_code', () => {
+      // Build HTML with duplicate articles
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+      const codes = units.map(u => u.unit_code);
+
+      // Should have 2 unique unit codes
+      expect(units).toHaveLength(2);
+      expect(new Set(codes).size).toBe(codes.length);
+    });
+
+    it('should keep the first occurrence when deduplicating', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // First occurrence (id=1, price=1000) should be kept
+      expect(units[0].id).toBe('1');
+      expect(units[0].price).toBe('1000');
+    });
+
+    it('should log a warning when duplicates are removed', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Duplicate units removed',
+        expect.objectContaining({
+          original: 3,
+          deduplicated: 2
+        })
+      );
+    });
+
+    it('should not log a warning when there are no duplicates', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      // The fixture has no duplicates, so no warn about duplicates
+      const warnCalls = mockLogger.warn.mock.calls;
+      const duplicateWarns = warnCalls.filter(
+        call => call[0] === 'Duplicate units removed'
+      );
+      expect(duplicateWarns).toHaveLength(0);
+    });
+
+    it('should filter out articles with no unit_code during deduplication', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // Article without unit_code should be filtered out
+      expect(units).toHaveLength(1);
+      expect(units[0].unit_code).toBe('VALID-1');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging behavior
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of parsed units', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 5 })
+      );
+    });
+
+    it('should log error when container is not found', () => {
+      const html = '<html><body><p>Nothing here</p></body></html>';
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.error).toHaveBeenCalledTimes(1);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Multiple units with varied data
+  // ---------------------------------------------------------------
+  describe('varied unit data', () => {
+    it('should handle unavailable units correctly', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit4 = units.find(u => u.unit_code === 'CCT-415');
+
+      expect(unit4.available).toBe('false');
+      expect(unit4.unavailable).toBe('true');
+      expect(unit4.soonest).toBe('');
+      expect(unit4.date_available).toBe('');
+    });
+
+    it('should handle half bath counts', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
+
+      expect(unit2.bath_count).toBe('1.5');
+    });
+
+    it('should handle studio units (bed_count = 0)', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
+
+      expect(unit3.bed_count).toBe('0');
+    });
+  });
+});

__tests__/scraper/recordScraperRun.test.js

@@ -0,0 +1,307 @@
+/**
+ * Tests for recordScraperRun() history persistence
+ *
+ * Covers:
+ * - insertOne is called with runData plus recordedAt timestamp
+ * - All runData fields are preserved (jobId, trigger, status, duration, etc.)
+ * - Error is caught and logged but NOT thrown (graceful failure)
+ * - Returns null on error instead of crashing
+ * - Returns insert result on success
+ * - recordedAt is a valid Date object
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require after implementation
+let recordScraperRun;
+
+let mongoServer;
+let client;
+let db;
+let logger;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ recordScraperRun } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+
+  // Create fresh mock logger for each test
+  logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+});
+
+describe('recordScraperRun', () => {
+  const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
+
+  // Sample runData matching the structure from runScrape()'s finally block
+  function createSampleRunData(overrides = {}) {
+    return {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      dryRun: false,
+      status: 'success',
+      startedAt: '2026-02-06T06:00:00.000Z',
+      completedAt: '2026-02-06T06:00:12.345Z',
+      duration: 12345,
+      unitsProcessed: 50,
+      pricesInserted: 48,
+      newUnitsCount: 3,
+      rentedUnitsCount: 1,
+      staleUnitsCount: 2,
+      errors: [],
+      ...overrides
+    };
+  }
+
+  // ---------------------------------------------------------------
+  // 1. insertOne is called with runData plus recordedAt timestamp
+  // ---------------------------------------------------------------
+  describe('insertOne with runData and recordedAt', () => {
+    it('should insert a document into the scraper_runs collection', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+
+    it('should include a recordedAt field in the inserted document', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. All runData fields are preserved
+  // ---------------------------------------------------------------
+  describe('preserving all runData fields', () => {
+    it('should preserve jobId, trigger, status, and duration', async () => {
+      const runData = createSampleRunData({
+        jobId: 'preserve-test-001',
+        trigger: 'scheduled',
+        status: 'failed',
+        duration: 9999
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.jobId).toBe('preserve-test-001');
+      expect(doc.trigger).toBe('scheduled');
+      expect(doc.status).toBe('failed');
+      expect(doc.duration).toBe(9999);
+    });
+
+    it('should preserve unit processing metrics', async () => {
+      const runData = createSampleRunData({
+        unitsProcessed: 42,
+        pricesInserted: 40,
+        newUnitsCount: 5,
+        rentedUnitsCount: 2,
+        staleUnitsCount: 3
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.unitsProcessed).toBe(42);
+      expect(doc.pricesInserted).toBe(40);
+      expect(doc.newUnitsCount).toBe(5);
+      expect(doc.rentedUnitsCount).toBe(2);
+      expect(doc.staleUnitsCount).toBe(3);
+    });
+
+    it('should preserve timing fields (startedAt, completedAt)', async () => {
+      const runData = createSampleRunData({
+        startedAt: '2026-02-06T10:00:00.000Z',
+        completedAt: '2026-02-06T10:00:15.500Z'
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
+      expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
+    });
+
+    it('should preserve dryRun flag', async () => {
+      const runData = createSampleRunData({ dryRun: true });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.dryRun).toBe(true);
+    });
+
+    it('should preserve errors array when it has entries', async () => {
+      const runData = createSampleRunData({
+        status: 'failed',
+        errors: ['Connection timeout', 'Retry exhausted']
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
+    });
+
+    it('should preserve empty errors array for successful runs', async () => {
+      const runData = createSampleRunData({ errors: [] });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Error is caught and logged but NOT thrown (graceful failure)
+  // ---------------------------------------------------------------
+  describe('graceful error handling', () => {
+    it('should NOT throw when insertOne fails', async () => {
+      const runData = createSampleRunData();
+
+      // Create a mock db that throws on insertOne
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      // This should NOT throw
+      await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
+    });
+
+    it('should log error message when insert fails', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await recordScraperRun(mockDb, runData, logger);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to record scraper run',
+        { errorMessage: 'Disk full' }
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Returns null on error instead of crashing
+  // ---------------------------------------------------------------
+  describe('return value on error', () => {
+    it('should return null when insertOne throws', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const result = await recordScraperRun(mockDb, runData, logger);
+      expect(result).toBeNull();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Returns insert result on success
+  // ---------------------------------------------------------------
+  describe('return value on success', () => {
+    it('should return the insertOne result object', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result).toBeDefined();
+      expect(result).not.toBeNull();
+    });
+
+    it('should return result with acknowledged property', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.acknowledged).toBe(true);
+    });
+
+    it('should return result with insertedId', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.insertedId).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. recordedAt is a valid Date object
+  // ---------------------------------------------------------------
+  describe('recordedAt field', () => {
+    it('should set recordedAt as a Date instance', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+
+    it('should set recordedAt close to current time', async () => {
+      const beforeTime = new Date();
+
+      const runData = createSampleRunData();
+      await recordScraperRun(db, runData, logger);
+
+      const afterTime = new Date();
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+      expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
+    });
+
+    it('should not overwrite any existing runData fields with recordedAt', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+
+      // Verify the original fields still exist alongside recordedAt
+      expect(doc.jobId).toBe(runData.jobId);
+      expect(doc.status).toBe(runData.status);
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+  });
+});

__tests__/scraper/scraperConfig.test.js

@@ -0,0 +1,192 @@
+/**
+ * Tests for config/scraper.js
+ * Tests scraper configuration constants and environment variable overrides
+ */
+
+describe('config/scraper', () => {
+  // Store original env values to restore after tests
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    // Clear all scraper-related env vars before each test
+    delete process.env.SCRAPER_SCHEDULE;
+    delete process.env.SCRAPER_TIMEZONE;
+    delete process.env.SCRAPER_ENABLED;
+    delete process.env.SCRAPER_TIMEOUT;
+    delete process.env.SCRAPER_UNITS_COLLECTION;
+    delete process.env.SCRAPER_PRICES_COLLECTION;
+    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
+    delete process.env.SCRAPER_RUNS_COLLECTION;
+
+    // Clear the require cache to reload config with fresh env
+    jest.resetModules();
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    process.env = originalEnv;
+  });
+
+  describe('TARGET_URL', () => {
+    test('should be set to the correct apartment listing URL', () => {
+      const config = require('../../config/scraper');
+      expect(config.TARGET_URL).toBe(
+        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
+      );
+    });
+  });
+
+  describe('SCRAPER_SCHEDULE', () => {
+    test('should default to "0 6 * * *" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
+    });
+  });
+
+  describe('SCRAPER_TIMEZONE', () => {
+    test('should default to "UTC" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+    });
+  });
+
+  describe('SCRAPER_ENABLED', () => {
+    test('should default to true when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be false when env var is set to "false"', () => {
+      process.env.SCRAPER_ENABLED = 'false';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(false);
+    });
+
+    test('should be true when env var is set to any other value', () => {
+      process.env.SCRAPER_ENABLED = 'true';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be true when env var is empty string', () => {
+      process.env.SCRAPER_ENABLED = '';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+  });
+
+  describe('SCRAPER_TIMEOUT', () => {
+    test('should default to 30000 when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(30000);
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEOUT = '60000';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(60000);
+    });
+
+    test('should parse string to integer', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
+      expect(config.SCRAPER_TIMEOUT).toBe(45000);
+    });
+  });
+
+  describe('USER_AGENT', () => {
+    test('should be a valid User-Agent string', () => {
+      const config = require('../../config/scraper');
+      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
+    });
+  });
+
+  describe('RETRY_CONFIG', () => {
+    test('should have maxRetries property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
+      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
+    });
+
+    test('should have baseDelay property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
+      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
+    });
+
+    test('should have timeout property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
+      expect(config.RETRY_CONFIG.timeout).toBe(30000);
+    });
+
+    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG.timeout).toBe(45000);
+    });
+  });
+
+  describe('COLLECTIONS', () => {
+    describe('default values', () => {
+      test('UNITS should default to "units_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
+      });
+
+      test('PRICES should default to "unit_prices_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
+      });
+
+      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
+      });
+
+      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
+      });
+    });
+
+    describe('env var overrides', () => {
+      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
+        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
+      });
+
+      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
+        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
+      });
+
+      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
+        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
+      });
+
+      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
+        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
+      });
+    });
+  });
+});

__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

__tests__/scraper/scraperService.test.js

@@ -0,0 +1,457 @@
+/**
+ * Tests for runScrape() orchestration function
+ *
+ * Covers:
+ * - Full workflow completes with mocked dependencies
+ * - Returns correct result structure with jobId, status, metrics
+ * - Handles dryRun option (skip DB writes)
+ * - Handles htmlContent option (use provided HTML)
+ * - Records scraper run to history on success
+ * - Records scraper run to history on failure
+ * - Catches and logs errors from fetchPage
+ * - Catches and logs errors from database operations
+ * - Calculates newUnitsCount and rentedUnitsCount correctly
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+let mongoServer;
+let client;
+let db;
+
+// We will require runScrape and recordScraperRun after implementation
+let runScrape;
+let recordScraperRun;
+
+// Store original module references so we can mock individual functions
+let scraperService;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  scraperService = require('../../services/scraperService');
+  ({ runScrape, recordScraperRun } = scraperService);
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all relevant collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+  // Reset all mocks
+  jest.restoreAllMocks();
+});
+
+// ============================================================
+// Helper: Create sample HTML with units
+// ============================================================
+function createSampleHtml(unitCodes) {
+  const articles = unitCodes.map(code => `
+    <article
+      data-spaces-id="100"
+      data-spaces-unit="${code}"
+      data-spaces-unit-id="200"
+      data-spaces-unit-floor="5"
+      data-spaces-sort-area="750"
+      data-spaces-sort-bed="1"
+      data-spaces-sort-bath="1"
+      data-spaces-sort-price="1500"
+      data-spaces-available="true"
+      data-spaces-unavailable="false"
+      data-spaces-soonest="Now"
+      data-spaces-sort-date="1700000000"
+      data-spaces-plan-id="10"
+      data-spaces-sort-plan-name="Studio"
+      data-spaces-obj="unit"
+      data-spaces-community="TestCommunity"
+      data-spaces-asset="1"
+      data-spaces-href="/unit/${code}"
+      data-spaces-inventory-href="/inventory/${code}"
+    >
+      <img src="http://example.com/${code}.jpg" />
+    </article>
+  `).join('\n');
+
+  return `
+    <html><body>
+      <section class="spaces__tab-unit">
+        ${articles}
+      </section>
+    </body></html>
+  `;
+}
+
+// ============================================================
+// Test: recordScraperRun
+// ============================================================
+describe('recordScraperRun', () => {
+  it('should insert a run record into scraper_runs collection', async () => {
+    const runData = {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      status: 'success',
+      startedAt: new Date().toISOString(),
+      completedAt: new Date().toISOString(),
+      duration: 1234,
+      unitsProcessed: 10,
+      pricesInserted: 8,
+      errors: []
+    };
+
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(db, runData, mockLogger);
+
+    expect(result).toBeTruthy();
+    expect(result.insertedId).toBeTruthy();
+
+    // Verify it was inserted
+    const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
+    expect(saved).toBeTruthy();
+    expect(saved.status).toBe('success');
+    expect(saved.recordedAt).toBeInstanceOf(Date);
+  });
+
+  it('should not throw when insert fails', async () => {
+    // Pass null db to cause an error
+    const runData = { jobId: 'test-fail', status: 'success' };
+
+    // Should not throw
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(null, runData, mockLogger);
+    expect(result).toBeNull();
+    expect(mockLogger.error).toHaveBeenCalled();
+  });
+});
+
+// ============================================================
+// Test: runScrape - Full workflow
+// ============================================================
+describe('runScrape', () => {
+  it('should complete full workflow with mocked dependencies', async () => {
+    const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
+
+    const result = await runScrape(db, {
+      trigger: 'manual',
+      jobId: 'test-full-workflow',
+      htmlContent: html
+    });
+
+    expect(result).toBeTruthy();
+    expect(result.jobId).toBe('test-full-workflow');
+    expect(result.status).toBe('success');
+    expect(result.trigger).toBe('manual');
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.errors).toEqual([]);
+  });
+
+  it('should return correct result structure with jobId, status, metrics', async () => {
+    const html = createSampleHtml(['UNIT-X']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-structure',
+      htmlContent: html
+    });
+
+    // Verify all expected fields exist
+    expect(result).toHaveProperty('jobId', 'test-structure');
+    expect(result).toHaveProperty('trigger', 'manual');
+    expect(result).toHaveProperty('dryRun', false);
+    expect(result).toHaveProperty('status', 'success');
+    expect(result).toHaveProperty('startedAt');
+    expect(result).toHaveProperty('completedAt');
+    expect(result).toHaveProperty('duration');
+    expect(result).toHaveProperty('unitsProcessed');
+    expect(result).toHaveProperty('pricesInserted');
+    expect(result).toHaveProperty('newUnitsCount');
+    expect(result).toHaveProperty('rentedUnitsCount');
+    expect(result).toHaveProperty('staleUnitsCount');
+    expect(result).toHaveProperty('errors');
+
+    // Verify types
+    expect(typeof result.duration).toBe('number');
+    expect(result.duration).toBeGreaterThanOrEqual(0);
+    expect(typeof result.startedAt).toBe('string');
+    expect(typeof result.completedAt).toBe('string');
+    expect(Array.isArray(result.errors)).toBe(true);
+  });
+
+  it('should generate a jobId when not provided', async () => {
+    const html = createSampleHtml(['UNIT-GEN']);
+
+    const result = await runScrape(db, {
+      htmlContent: html
+    });
+
+    expect(result.jobId).toBeTruthy();
+    expect(typeof result.jobId).toBe('string');
+    // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+    expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
+  });
+
+  // ============================================================
+  // Test: dryRun option
+  // ============================================================
+  it('should skip DB writes when dryRun is true', async () => {
+    const html = createSampleHtml(['DRY-A', 'DRY-B']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-dry-run',
+      htmlContent: html,
+      dryRun: true
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.dryRun).toBe(true);
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.pricesInserted).toBe(0);
+    expect(result.staleUnitsCount).toBe(0);
+
+    // Verify no units were written to the units collection
+    const unitsCount = await db.collection('units_migration_test').countDocuments();
+    expect(unitsCount).toBe(0);
+
+    // Verify no prices were written
+    const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
+    expect(pricesCount).toBe(0);
+  });
+
+  // ============================================================
+  // Test: htmlContent option
+  // ============================================================
+  it('should use provided HTML instead of fetching when htmlContent is given', async () => {
+    const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
+
+    // Mock axios.get to track if HTTP fetch is attempted
+    const axios = require('axios');
+    const axiosSpy = jest.spyOn(axios, 'get');
+
+    const result = await runScrape(db, {
+      jobId: 'test-html-content',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(3);
+    // axios.get should NOT have been called since we provided htmlContent
+    expect(axiosSpy).not.toHaveBeenCalled();
+  });
+
+  // ============================================================
+  // Test: Records scraper run on success
+  // ============================================================
+  it('should record scraper run to history on success', async () => {
+    const html = createSampleHtml(['REC-A']);
+
+    await runScrape(db, {
+      jobId: 'test-record-success',
+      htmlContent: html
+    });
+
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('success');
+    expect(runRecord.recordedAt).toBeInstanceOf(Date);
+    expect(runRecord.unitsProcessed).toBe(1);
+  });
+
+  // ============================================================
+  // Test: Records scraper run on failure
+  // ============================================================
+  it('should record scraper run to history on failure', async () => {
+    const html = createSampleHtml(['FAIL-A']);
+
+    // Create a db proxy that throws on bulkWrite (used by upsertUnits)
+    // but allows other operations (like scraper_runs insertOne) to pass through
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'units_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Database connection lost');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-record-failure',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Database connection lost');
+
+    // The run should still be recorded (via the real db passed through for scraper_runs)
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+    expect(runRecord.errors).toContain('Database connection lost');
+  });
+
+  // ============================================================
+  // Test: Catches errors from fetchPage
+  // ============================================================
+  it('should catch and handle errors from fetchPage', async () => {
+    // Mock axios.get to throw a network error (fetchPage uses axios internally)
+    const axios = require('axios');
+    jest.spyOn(axios, 'get').mockRejectedValue(
+      new Error('Network timeout')
+    );
+
+    const result = await runScrape(db, {
+      jobId: 'test-fetch-error'
+      // No htmlContent, so it will call fetchPage which uses axios
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Network timeout');
+    expect(result.unitsProcessed).toBe(0);
+  });
+
+  // ============================================================
+  // Test: Catches errors from database operations
+  // ============================================================
+  it('should catch and handle errors from database operations', async () => {
+    const html = createSampleHtml(['DB-ERR']);
+
+    // Create a db proxy that throws on the prices collection bulkWrite
+    // Use Proxy to properly delegate all methods to the real collection
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'unit_prices_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Write concern timeout');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-db-error',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Write concern timeout');
+
+    // Run should still be recorded despite error
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+  });
+
+  // ============================================================
+  // Test: Calculates newUnitsCount and rentedUnitsCount correctly
+  // ============================================================
+  it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
+    // First, simulate yesterday's data by inserting price records for yesterday
+    const today = new Date();
+    const yesterday = new Date(today);
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const yesterdayStr = yesterday.toISOString().split('T')[0];
+
+    // Yesterday had units: OLD-A, OLD-B, OLD-C
+    await db.collection('unit_prices_migration_test').insertMany([
+      { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
+      { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
+      { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
+    ]);
+
+    // Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
+    const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-calc-changes',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    // NEW-D is new (not in yesterday's data)
+    expect(result.newUnitsCount).toBe(1);
+    // OLD-C was in yesterday's data but not in today's scrape
+    expect(result.rentedUnitsCount).toBe(1);
+    expect(result.unitsProcessed).toBe(3);
+  });
+
+  // ============================================================
+  // Test: Default trigger is 'manual'
+  // ============================================================
+  it('should default trigger to manual', async () => {
+    const html = createSampleHtml(['DEF-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-default-trigger',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('manual');
+  });
+
+  // ============================================================
+  // Test: Supports scheduled trigger
+  // ============================================================
+  it('should support scheduled trigger', async () => {
+    const html = createSampleHtml(['SCHED-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-scheduled-trigger',
+      trigger: 'scheduled',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('scheduled');
+  });
+
+  // ============================================================
+  // Test: Handles empty HTML (no units found)
+  // ============================================================
+  it('should handle HTML with no units gracefully', async () => {
+    const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+
+    const result = await runScrape(db, {
+      jobId: 'test-empty-html',
+      htmlContent: emptyHtml
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(0);
+    expect(result.errors).toContain('No units found in HTML');
+  });
+});

__tests__/scraper/updateDailySummary.test.js

@@ -0,0 +1,561 @@
+/**
+ * Tests for updateDailySummary() daily summary aggregation
+ *
+ * Covers:
+ * - Upsert by date key (YYYY-MM-DD)
+ * - Summary includes new_units, rented_units, stale_units arrays
+ * - Count fields are calculated correctly
+ * - net_change = newUnits.length - rentedUnits.length
+ * - total_available_today is set correctly
+ * - total_available_yesterday fetched from previous day
+ * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+ * - turnover_rate is 0 when yesterdayTotal is 0
+ * - Error handling when updateOne fails
+ * - Logging of summary update details
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require updateDailySummary after implementation
+let updateDailySummary;
+
+let mongoServer;
+let client;
+let db;
+
+const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ updateDailySummary } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('updateDailySummary', () => {
+  // ---------------------------------------------------------------
+  // 1. Upsert by date key
+  // ---------------------------------------------------------------
+  describe('upsert by date key', () => {
+    it('should insert a new summary when no document exists for the date', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 2,
+        totalAvailable: 15
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have upserted (created new)
+      expect(result.upsertedCount).toBe(1);
+
+      // Verify the document in the database
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc).not.toBeNull();
+      expect(doc.date).toBe('2026-02-05');
+    });
+
+    it('should update an existing summary for the same date (idempotent upsert)', async () => {
+      const logger = createMockLogger();
+
+      // Insert initial summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-05',
+        new_units: ['A101'],
+        rented_units: [],
+        total_available_today: 10
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 12
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have matched and modified (not upserted)
+      expect(result.matchedCount).toBe(1);
+
+      // Verify updated values
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
+      expect(doc.total_available_today).toBe(12);
+
+      // Verify only one document exists for this date
+      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Summary includes new_units, rented_units, stale_units arrays
+  // ---------------------------------------------------------------
+  describe('summary arrays', () => {
+    it('should store new_units, rented_units, and stale_units arrays', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202', 'B203'],
+        staleUnitsCount: 3,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102']);
+      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
+      expect(doc.stale_units).toEqual([]);
+    });
+
+    it('should handle empty arrays for new_units and rented_units', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 25
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual([]);
+      expect(doc.rented_units).toEqual([]);
+      expect(doc.stale_units).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Count fields are calculated correctly
+  // ---------------------------------------------------------------
+  describe('count fields', () => {
+    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 5,
+        totalAvailable: 30
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(3);
+      expect(doc.rented_units_count).toBe(1);
+      expect(doc.stale_units_count).toBe(5);
+    });
+
+    it('should set all count fields to 0 when arrays are empty', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(0);
+      expect(doc.rented_units_count).toBe(0);
+      expect(doc.stale_units_count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. net_change = newUnits.length - rentedUnits.length
+  // ---------------------------------------------------------------
+  describe('net_change calculation', () => {
+    it('should calculate positive net_change when more units added than rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(2); // 3 - 1 = 2
+    });
+
+    it('should calculate negative net_change when more units rented than added', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
+    });
+
+    it('should calculate zero net_change when equal new and rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. total_available_today is set correctly
+  // ---------------------------------------------------------------
+  describe('total_available_today', () => {
+    it('should store the totalAvailable value as total_available_today', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 42
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(42);
+    });
+
+    it('should handle totalAvailable of 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 0
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. total_available_yesterday fetched from previous day
+  // ---------------------------------------------------------------
+  describe('total_available_yesterday', () => {
+    it('should fetch total_available_today from previous day summary', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 35,
+        new_units: [],
+        rented_units: []
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 35
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(35);
+    });
+
+    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(0);
+    });
+
+    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
+      const logger = createMockLogger();
+
+      // Insert March 31 summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-03-31',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-04-01',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 48
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
+      expect(doc.total_available_yesterday).toBe(50);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+  // ---------------------------------------------------------------
+  describe('turnover_rate calculation', () => {
+    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 50 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
+        staleUnitsCount: 0,
+        totalAvailable: 46
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 5 rented / 50 yesterday = 10%
+      expect(doc.turnover_rate).toBe(10);
+    });
+
+    it('should round turnover_rate to 2 decimal places', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 30 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 30
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 28
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
+      expect(doc.turnover_rate).toBe(6.67);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. turnover_rate is 0 when yesterdayTotal is 0
+  // ---------------------------------------------------------------
+  describe('turnover_rate edge cases', () => {
+    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 0 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 0
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Error handling when updateOne fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw and log error when updateOne fails', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      // Create a mock db that throws on updateOne
+      const mockCollection = {
+        findOne: jest.fn().mockResolvedValue(null),
+        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(
+        updateDailySummary(mockDb, summaryData, logger)
+      ).rejects.toThrow('Write concern timeout');
+
+      // Verify error was logged
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to update daily summary',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Logging of summary update details
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log summary update details on success', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 18
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Daily summary updated',
+        expect.objectContaining({
+          date: '2026-02-05',
+          newUnits: 2,
+          rentedUnits: 1,
+          totalAvailable: 18
+        })
+      );
+    });
+
+    it('should include timestamp field in the stored summary', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.timestamp).toBeDefined();
+      expect(typeof doc.timestamp).toBe('string');
+      // Verify it is a valid ISO timestamp
+      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
+    });
+  });
+});

__tests__/scraper/upsertUnits.test.js

@@ -0,0 +1,498 @@
+/**
+ * Tests for upsertUnits() bulk operation
+ *
+ * Covers:
+ * - bulkWrite is called with correct updateOne operations
+ * - upsert: true is set for all operations
+ * - $set updates last_scraped and data_source fields
+ * - $setOnInsert sets first_seen for new units
+ * - Empty units array logs warning and returns early
+ * - ordered: false is set for parallel execution
+ * - Error handling when bulkWrite fails
+ * - Logging of matched, modified, and upserted counts
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// We will require upsertUnits after implementation
+let upsertUnits;
+
+let mongoServer;
+let client;
+let db;
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ upsertUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the units collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('upsertUnits', () => {
+  const UNITS_COLLECTION = 'units_migration_test';
+
+  // ---------------------------------------------------------------
+  // 1. bulkWrite is called with correct updateOne operations
+  // ---------------------------------------------------------------
+  describe('bulkWrite operations', () => {
+    it('should create updateOne operations for each unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
+        { unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.upsertedCount).toBe(2);
+
+      // Verify documents exist in the collection
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'B202']);
+    });
+
+    it('should filter by unit_code as the unique key', async () => {
+      const logger = createMockLogger();
+
+      // Insert a unit first
+      const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
+      await upsertUnits(db, units, logger);
+
+      // Upsert again with updated price
+      const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
+      const result = await upsertUnits(db, updatedUnits, logger);
+
+      // Should match and modify, not insert new
+      expect(result.matchedCount).toBe(1);
+      expect(result.upsertedCount).toBe(0);
+
+      // Verify only one document exists
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+      expect(docs[0].price).toBe(1400);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. upsert: true is set for all operations
+  // ---------------------------------------------------------------
+  describe('upsert behavior', () => {
+    it('should insert new units when they do not exist (upsert: true)', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'NEW-001', price: 1100 },
+        { unit_code: 'NEW-002', price: 1200 },
+        { unit_code: 'NEW-003', price: 1300 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(3);
+
+      // Verify all three are in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+
+    it('should update existing units without creating duplicates', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
+
+      // Second upsert with same unit_code
+      await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. $set updates last_scraped and data_source fields
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set last_scraped timestamp on every upsert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'T100', price: 1000 }];
+      await upsertUnits(db, units, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
+      expect(doc.last_scraped).toBeDefined();
+      expect(typeof doc.last_scraped).toBe('string');
+      // The last_scraped should be between before and after timestamps
+      expect(doc.last_scraped >= beforeTime).toBe(true);
+      expect(doc.last_scraped <= afterTime).toBe(true);
+    });
+
+    it('should set data_source to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'T200', price: 1100 }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should update last_scraped on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const firstScraped = doc1.last_scraped;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const secondScraped = doc2.last_scraped;
+
+      expect(secondScraped > firstScraped).toBe(true);
+    });
+
+    it('should spread all unit fields into the $set operation', async () => {
+      const logger = createMockLogger();
+      const units = [{
+        unit_code: 'T400',
+        price: 1500,
+        bed_count: 2,
+        bath_count: 1.5,
+        area: 900,
+        floor: 3,
+        plan_name: 'Luxury',
+        community: 'Tower A'
+      }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
+      expect(doc.price).toBe(1500);
+      expect(doc.bed_count).toBe(2);
+      expect(doc.bath_count).toBe(1.5);
+      expect(doc.area).toBe(900);
+      expect(doc.floor).toBe(3);
+      expect(doc.plan_name).toBe('Luxury');
+      expect(doc.community).toBe('Tower A');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. $setOnInsert sets first_seen for new units
+  // ---------------------------------------------------------------
+  describe('$setOnInsert for first_seen', () => {
+    it('should set first_seen on initial insert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
+      expect(doc.first_seen).toBeDefined();
+      expect(typeof doc.first_seen).toBe('string');
+      expect(doc.first_seen >= beforeTime).toBe(true);
+    });
+
+    it('should NOT update first_seen on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+      const originalFirstSeen = doc1.first_seen;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert (update)
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+
+      // first_seen should remain unchanged
+      expect(doc2.first_seen).toBe(originalFirstSeen);
+    });
+
+    it('should set different first_seen for different units inserted at different times', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
+
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
+
+      // Both should have first_seen, but different values
+      expect(doc1.first_seen).toBeDefined();
+      expect(doc2.first_seen).toBeDefined();
+      expect(doc2.first_seen > doc1.first_seen).toBe(true);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty units array logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty units array', () => {
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
+    });
+
+    it('should return early with zero counts for empty array', async () => {
+      const logger = createMockLogger();
+
+      const result = await upsertUnits(db, [], logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(result.upsertedCount).toBe(0);
+    });
+
+    it('should not perform any database writes for empty array', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. ordered: false is set for parallel execution
+  // ---------------------------------------------------------------
+  describe('ordered: false for parallel execution', () => {
+    it('should successfully process all units even if one has a conflict', async () => {
+      const logger = createMockLogger();
+
+      // Insert multiple units - ordered: false means all operations execute
+      // even if some fail. We verify by checking all valid units are present.
+      const units = [
+        { unit_code: 'P100', price: 1000 },
+        { unit_code: 'P200', price: 1100 },
+        { unit_code: 'P300', price: 1200 },
+        { unit_code: 'P400', price: 1300 },
+        { unit_code: 'P500', price: 1400 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(5);
+
+      // All 5 units should be in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+
+    it('should handle a mix of new inserts and existing updates', async () => {
+      const logger = createMockLogger();
+
+      // First, insert some units
+      await upsertUnits(db, [
+        { unit_code: 'M100', price: 900 },
+        { unit_code: 'M200', price: 1000 }
+      ], logger);
+
+      // Now upsert a mix of existing and new units
+      const mixedUnits = [
+        { unit_code: 'M100', price: 950 },  // existing - update
+        { unit_code: 'M200', price: 1050 }, // existing - update
+        { unit_code: 'M300', price: 1100 }, // new - insert
+      ];
+
+      const result = await upsertUnits(db, mixedUnits, logger);
+
+      expect(result.matchedCount).toBe(2);
+      expect(result.upsertedCount).toBe(1);
+
+      // Total should be 3 documents
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E100', price: 1000 }];
+
+      // Create a mock db that throws on bulkWrite
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E200', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await upsertUnits(mockDb, units, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to upsert units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E300', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Logging of matched, modified, and upserted counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log matched, modified, and upserted counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L100', price: 1000 },
+        { unit_code: 'L200', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: expect.any(Number),
+          modified: expect.any(Number),
+          upserted: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L300', price: 1000 },
+        { unit_code: 'L400', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      // For new inserts: matched=0, modified=0, upserted=2
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 0,
+          modified: 0,
+          upserted: 2
+        })
+      );
+    });
+
+    it('should log correct counts for updates to existing units', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
+
+      // Reset mock to capture only the second call
+      logger.info.mockClear();
+
+      // Update existing
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 1,
+          modified: 1,
+          upserted: 0
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the bulkWrite result object', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'R100', price: 1000 }];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // bulkWrite result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('upsertedCount');
+    });
+  });
+});

__tests__/setup.js

@@ -0,0 +1,13 @@
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+module.exports = async () => {
+  // Create an in-memory MongoDB instance for testing
+  const mongod = await MongoMemoryServer.create();
+  const uri = mongod.getUri();
+
+  // Store the URI and instance globally so tests can access them
+  global.__MONGOD__ = mongod;
+  process.env.MONGO_URI = uri;
+  process.env.JWT_SECRET = 'test-jwt-secret-for-testing-only';
+  process.env.NODE_ENV = 'test';
+};

__tests__/setupAfterEnv.js

@@ -0,0 +1,11 @@
+// Increase timeout for tests that interact with MongoDB
+jest.setTimeout(30000);
+
+// Suppress console logs during tests (optional - comment out for debugging)
+// global.console = {
+//   ...console,
+//   log: jest.fn(),
+//   debug: jest.fn(),
+//   info: jest.fn(),
+//   warn: jest.fn(),
+// };

__tests__/teardown.js

@@ -0,0 +1,6 @@
+module.exports = async () => {
+  // Stop the in-memory MongoDB instance
+  if (global.__MONGOD__) {
+    await global.__MONGOD__.stop();
+  }
+};

config/auth.js

@@ -0,0 +1,24 @@
+// Google OAuth and JWT configuration
+module.exports = {
+  google: {
+    clientID: process.env.GOOGLE_CLIENT_ID,
+    clientSecret: process.env.GOOGLE_CLIENT_SECRET,
+    callbackURL: process.env.GOOGLE_CALLBACK_URL,
+    scope: ['profile', 'email']
+  },
+  jwt: {
+    secret: process.env.JWT_SECRET,
+    expiresIn: '7d',
+    refreshThreshold: 24 * 60 * 60  // Refresh if token is older than 1 day (in seconds)
+  },
+  cookie: {
+    name: 'auth_token',
+    options: {
+      httpOnly: true,
+      secure: process.env.NODE_ENV === 'production',
+      sameSite: 'lax',
+      maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
+      domain: process.env.NODE_ENV === 'production' ? '.maverickapplications.com' : undefined
+    }
+  }
+};

config/scraper.js

@@ -0,0 +1,40 @@
+/**
+ * Scraper Configuration Module
+ *
+ * Contains all constants and settings for the apartment price scraper.
+ * All values can be overridden via environment variables for deployment flexibility.
+ *
+ * See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
+ */
+
+module.exports = {
+  // Target URL for apartment listings
+  TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
+
+  // Scheduling configuration
+  SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
+  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
+  SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
+
+  // HTTP settings
+  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
+
+  // Retry configuration for HTTP requests
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
+    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
+  },
+
+  // Graceful shutdown timeout (how long to wait for running job before force-stopping)
+  SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
+
+  // MongoDB collection names (environment variable overrides for development isolation)
+  COLLECTIONS: {
+    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
+    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
+    DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
+    SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
+  }
+};

cosign.pub

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEBOFl4SHzoZVM59+668t2tecTyKM+
+cE4zaOkbFO30u7d2+bjpwPnbYrTutKmAeehYzCAVW/OYea7ML0cW2YCSCQ==
+-----END PUBLIC KEY-----

docker-compose.yml

@@ -1,12 +1,21 @@
 services:
   apartment-api:
+    image: ${IMAGE:-apartment-api:latest}
     build: .
     container_name: apartment-api
     restart: unless-stopped
+    env_file:
+      - .env
     environment:
       - NODE_ENV=production
-      - PORT=8080  # Changed from 3000 to 8080
-      - MONGO_URI=mongodb://admin:password123@mongodb:27017
+      - PORT=8080
+      - MONGO_URI=${MONGO_URI}
+      - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID}
+      - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET}
+      - GOOGLE_CALLBACK_URL=${GOOGLE_CALLBACK_URL}
+      - JWT_SECRET=${JWT_SECRET}
+      - FRONTEND_URL=${FRONTEND_URL}
+      - ACTIVITY_LOG_LEVEL=${ACTIVITY_LOG_LEVEL:-all}
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=traefik"

eslint.config.mjs

@@ -0,0 +1,34 @@
+import js from '@eslint/js';
+import globals from 'globals';
+
+export default [
+  js.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'commonjs',
+      globals: {
+        ...globals.node,
+        ...globals.jest,
+      },
+    },
+    rules: {
+      // Allow unused vars with underscore prefix, and unused function args
+      'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
+      'no-console': 'off',
+      // Formatting rules as warnings (won't fail CI, but will be reported)
+      'semi': ['warn', 'always'],
+      'no-multiple-empty-lines': ['warn', { max: 2 }],
+      'eol-last': ['warn', 'always'],
+      'no-trailing-spaces': 'warn',
+    },
+  },
+  {
+    ignores: [
+      'node_modules/**',
+      'coverage/**',
+      '*.min.js',
+      'eslint.config.mjs',
+    ],
+  },
+];

jest.config.js

@@ -0,0 +1,17 @@
+module.exports = {
+  testEnvironment: 'node',
+  testMatch: ['**/__tests__/**/*.test.js'],
+  collectCoverageFrom: [
+    '**/*.js',
+    '!**/node_modules/**',
+    '!**/coverage/**',
+    '!jest.config.js'
+  ],
+  coverageDirectory: 'coverage',
+  verbose: true,
+  testTimeout: 30000,
+  // Setup file to handle MongoDB memory server lifecycle
+  globalSetup: './__tests__/setup.js',
+  globalTeardown: './__tests__/teardown.js',
+  setupFilesAfterEnv: ['./__tests__/setupAfterEnv.js']
+};

jobs/scraperJob.js

@@ -0,0 +1,275 @@
+const cron = require('node-cron');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { runScrape } = require('../services/scraperService');
+const { createLogger } = require('../services/scraperLogger');
+
+// In-process mutex state
+let isRunning = false;
+let currentJobId = null;
+
+// Scheduler state
+let scheduledJob = null;
+
+// Shutdown state
+let shuttingDown = false;
+let runningJobPromise = null;
+
+/**
+ * Check if scraper is currently running
+ * @returns {boolean}
+ */
+function isScraperRunning() {
+  return isRunning;
+}
+
+/**
+ * Get current job ID if running
+ * @returns {string|null}
+ */
+function getCurrentJobId() {
+  return currentJobId;
+}
+
+/**
+ * Acquire the scraper lock
+ * @param {string} jobId - Job ID to set
+ * @returns {boolean} True if lock acquired
+ */
+function acquireLock(jobId) {
+  if (isRunning || shuttingDown) {
+    return false;
+  }
+  isRunning = true;
+  currentJobId = jobId;
+  return true;
+}
+
+/**
+ * Release the scraper lock
+ */
+function releaseLock() {
+  isRunning = false;
+  currentJobId = null;
+}
+
+/**
+ * Get the configured schedule expression
+ * @returns {string} Cron expression or 'disabled'
+ */
+function getScheduleExpression() {
+  if (!config.SCRAPER_ENABLED) {
+    return 'disabled';
+  }
+  return config.SCRAPER_SCHEDULE;
+}
+
+/**
+ * Calculate next scheduled run time
+ * @returns {string|null} ISO timestamp or null if disabled
+ */
+function getNextScheduledRun() {
+  if (!config.SCRAPER_ENABLED || !scheduledJob) {
+    return null;
+  }
+
+  const { CronExpressionParser } = require('cron-parser');
+  try {
+    const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
+      tz: config.SCRAPER_TIMEZONE
+    });
+    return interval.next().toISOString();
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Check if schedule runs more frequently than 1 hour
+ * @param {string} schedule - Cron expression
+ * @returns {boolean} True if schedule is too frequent
+ */
+function isScheduleTooFrequent(schedule) {
+  const parts = schedule.trim().split(/\s+/);
+
+  if (parts.length < 5) return false;
+
+  const minuteField = parts[0];
+
+  // If minute field is */N with N < 60, it runs more than once per hour
+  if (/^\*\/\d+$/.test(minuteField)) {
+    const interval = parseInt(minuteField.substring(2), 10);
+    if (interval < 60) return true;
+  }
+
+  // If minute field is *, it runs every minute
+  if (minuteField === '*') return true;
+
+  return false;
+}
+
+/**
+ * Initialize the scraper scheduler
+ * @param {Db} db - MongoDB database instance
+ */
+function initializeScheduler(db) {
+  const logger = createLogger('scheduler');
+
+  // Check if scheduling is enabled
+  if (!config.SCRAPER_ENABLED) {
+    logger.info('Scraper scheduling is disabled');
+    return;
+  }
+
+  // Validate cron expression
+  if (!cron.validate(config.SCRAPER_SCHEDULE)) {
+    logger.error('Invalid cron schedule expression', {
+      schedule: config.SCRAPER_SCHEDULE
+    });
+    logger.warn('Falling back to default schedule: 0 6 * * *');
+    config.SCRAPER_SCHEDULE = '0 6 * * *';
+  }
+
+  // Validate minimum interval (1 hour)
+  if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
+    logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
+      originalSchedule: config.SCRAPER_SCHEDULE
+    });
+    config.SCRAPER_SCHEDULE = '0 * * * *';
+  }
+
+  // Create the scheduled job
+  scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
+    const jobId = crypto.randomUUID();
+    const jobLogger = createLogger(jobId);
+
+    jobLogger.info('Scheduled scrape triggered');
+
+    // Check if already running or shutting down
+    if (!acquireLock(jobId)) {
+      jobLogger.warn('Skipped - scrape already in progress');
+      return;
+    }
+
+    try {
+      const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
+      runningJobPromise = jobExecution;
+      await jobExecution;
+    } catch (error) {
+      jobLogger.error('Scheduled scrape failed', {
+        errorType: error.name,
+        errorMessage: error.message
+      });
+    } finally {
+      runningJobPromise = null;
+      releaseLock();
+    }
+  }, {
+    timezone: config.SCRAPER_TIMEZONE,
+    scheduled: true
+  });
+
+  logger.info('Scraper scheduler initialized', {
+    schedule: config.SCRAPER_SCHEDULE,
+    timezone: config.SCRAPER_TIMEZONE,
+    nextRun: getNextScheduledRun()
+  });
+}
+
+/**
+ * Stop the scheduler (for graceful shutdown)
+ */
+function stopScheduler() {
+  if (scheduledJob) {
+    scheduledJob.stop();
+    scheduledJob = null;
+  }
+}
+
+/**
+ * Check if the scraper is in the process of shutting down
+ * @returns {boolean}
+ */
+function isShuttingDown() {
+  return shuttingDown;
+}
+
+/**
+ * Perform a graceful shutdown of the scraper
+ * - Stops the cron scheduler to prevent new jobs
+ * - Waits for any running job to complete (with timeout)
+ * - Releases the mutex lock
+ * - Logs shutdown progress
+ * @returns {Promise<void>}
+ */
+async function gracefulShutdown() {
+  const logger = createLogger('shutdown');
+
+  logger.info('Shutdown initiated');
+
+  // Mark as shutting down to prevent new jobs
+  shuttingDown = true;
+
+  // Stop the cron scheduler
+  stopScheduler();
+
+  // Wait for running job to complete (with timeout)
+  if (isRunning && runningJobPromise) {
+    logger.info('Waiting for running job to complete', {
+      jobId: currentJobId,
+      timeout: config.SHUTDOWN_TIMEOUT
+    });
+
+    let timeoutHandle;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
+    });
+
+    const result = await Promise.race([
+      runningJobPromise.then(() => 'completed').catch(() => 'completed'),
+      timeoutPromise
+    ]);
+
+    clearTimeout(timeoutHandle);
+
+    if (result === 'timeout') {
+      logger.warn('Shutdown wait for running job timed out', {
+        jobId: currentJobId,
+        timeout: config.SHUTDOWN_TIMEOUT
+      });
+    }
+  }
+
+  // Force release the lock
+  releaseLock();
+
+  logger.info('Shutdown complete');
+}
+
+/**
+ * Register process signal handlers for graceful shutdown
+ * Listens for SIGTERM and SIGINT signals
+ */
+function registerSignalHandlers() {
+  process.on('SIGTERM', () => {
+    gracefulShutdown();
+  });
+
+  process.on('SIGINT', () => {
+    gracefulShutdown();
+  });
+}
+
+module.exports = {
+  isScraperRunning,
+  getCurrentJobId,
+  acquireLock,
+  releaseLock,
+  getScheduleExpression,
+  getNextScheduledRun,
+  initializeScheduler,
+  stopScheduler,
+  isShuttingDown,
+  gracefulShutdown,
+  registerSignalHandlers
+};

middleware/auth.js

@@ -0,0 +1,111 @@
+const jwt = require('jsonwebtoken');
+const authConfig = require('../config/auth');
+const { findById } = require('../models/user');
+
+/**
+ * Check if a token should be refreshed based on its age
+ * @param {Object} decoded - Decoded JWT payload
+ * @returns {boolean} True if token should be refreshed
+ */
+const shouldRefreshToken = (decoded) => {
+  const tokenAge = Math.floor(Date.now() / 1000) - decoded.iat;
+  return tokenAge > authConfig.jwt.refreshThreshold;
+};
+
+/**
+ * Generate a new JWT token for a user
+ * @param {string|ObjectId} userId - User's MongoDB _id
+ * @returns {string} JWT token
+ */
+const generateToken = (userId) => {
+  return jwt.sign(
+    { userId: userId.toString() },
+    authConfig.jwt.secret,
+    { expiresIn: authConfig.jwt.expiresIn }
+  );
+};
+
+/**
+ * Authentication middleware
+ * Validates JWT token from cookie and attaches user to request
+ * Implements sliding window token refresh
+ *
+ * @param {Request} req - Express request object
+ * @param {Response} res - Express response object
+ * @param {Function} next - Express next function
+ */
+const requireAuth = async (req, res, next) => {
+  const token = req.cookies[authConfig.cookie.name];
+
+  // No token present
+  if (!token) {
+    return res.status(401).json({ error: 'Authentication required' });
+  }
+
+  try {
+    // Verify the token
+    const decoded = jwt.verify(token, authConfig.jwt.secret);
+
+    // Get database instance
+    const db = req.app.locals.db;
+
+    // Fetch user from database
+    const user = await findById(db, decoded.userId);
+
+    // User not found
+    if (!user) {
+      res.clearCookie(authConfig.cookie.name);
+      return res.status(401).json({ error: 'Authentication required' });
+    }
+
+    // User is disabled (silent logout)
+    if (!user.isActive) {
+      res.clearCookie(authConfig.cookie.name);
+      return res.status(401).json({ error: 'Authentication required' });
+    }
+
+    // Attach user to request
+    req.user = user;
+
+    // Sliding window token refresh
+    if (shouldRefreshToken(decoded)) {
+      const newToken = generateToken(user._id);
+      res.cookie(authConfig.cookie.name, newToken, authConfig.cookie.options);
+    }
+
+    next();
+  } catch (err) {
+    // Token verification failed (invalid or expired)
+    res.clearCookie(authConfig.cookie.name);
+    return res.status(401).json({ error: 'Invalid token' });
+  }
+};
+
+/**
+ * Admin authorization middleware
+ * Must be used after requireAuth middleware
+ * Checks if authenticated user has admin role
+ *
+ * @param {Request} req - Express request object
+ * @param {Response} res - Express response object
+ * @param {Function} next - Express next function
+ */
+const requireAdmin = (req, res, next) => {
+  // Check if user is attached (requireAuth should be called first)
+  if (!req.user) {
+    return res.status(401).json({ error: 'Authentication required' });
+  }
+
+  // Check if user has admin role
+  if (req.user.role !== 'admin') {
+    return res.status(403).json({ error: 'Admin access required' });
+  }
+
+  next();
+};
+
+module.exports = {
+  requireAuth,
+  requireAdmin,
+  generateToken
+};

middleware/passport.js

@@ -0,0 +1,36 @@
+const passport = require('passport');
+const GoogleStrategy = require('passport-google-oauth20').Strategy;
+const authConfig = require('../config/auth');
+const { findOrCreateUser } = require('../models/user');
+
+const configurePassport = (passport, db) => {
+  passport.use(new GoogleStrategy({
+    clientID: authConfig.google.clientID,
+    clientSecret: authConfig.google.clientSecret,
+    callbackURL: authConfig.google.callbackURL
+  },
+  async (accessToken, refreshToken, profile, done) => {
+    try {
+      // Safely extract email
+      const email = profile.emails?.[0]?.value;
+      if (!email) {
+        return done(new Error('No email found in Google profile'), null);
+      }
+
+      const userProfile = {
+        googleId: profile.id,
+        email: email,
+        name: profile.displayName,
+        picture: profile.photos?.[0]?.value || null
+      };
+
+      const user = await findOrCreateUser(db, userProfile);
+      done(null, user);
+    } catch (error) {
+      console.error('Passport strategy error:', error);
+      done(error, null);
+    }
+  }));
+};
+
+module.exports = { configurePassport };

models/user.js

@@ -0,0 +1,253 @@
+const { ObjectId } = require('mongodb');
+
+const USER_COLLECTION = 'users';
+
+/**
+ * Find or create a user based on Google OAuth profile
+ * Uses upsert pattern to handle both new and returning users
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} profile - Google OAuth profile
+ * @param {string} profile.id - Google's unique user ID
+ * @param {string} profile.displayName - User's display name
+ * @param {Array} profile.emails - Array of email objects
+ * @param {Array} profile.photos - Array of photo objects
+ * @returns {Promise<Object>} User document
+ */
+async function findOrCreateUser(db, profile) {
+  const now = new Date();
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { googleId: profile.googleId },
+    {
+      $set: {
+        email: profile.email,
+        name: profile.name,
+        picture: profile.picture || null,
+        lastLoginAt: now
+      },
+      $inc: { loginCount: 1 },
+      $setOnInsert: {
+        googleId: profile.googleId,
+        isActive: true,
+        role: 'user',
+        createdAt: now
+      }
+    },
+    {
+      upsert: true,
+      returnDocument: 'after'
+    }
+  );
+
+  return result;
+}
+
+/**
+ * Find a user by Google ID
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string} googleId - Google's unique user ID
+ * @returns {Promise<Object|null>} User document or null
+ */
+async function findByGoogleId(db, googleId) {
+  return await db.collection(USER_COLLECTION).findOne({ googleId });
+}
+
+/**
+ * Find a user by MongoDB ObjectId
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @returns {Promise<Object|null>} User document or null
+ */
+async function findById(db, id) {
+  // Convert string to ObjectId if needed
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+  return await db.collection(USER_COLLECTION).findOne({ _id: objectId });
+}
+
+/**
+ * Enable or disable a user account
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @param {boolean} isActive - New active status
+ * @param {string|ObjectId|null} adminId - Admin performing the action (required when disabling)
+ * @returns {Promise<Object>} Update result
+ */
+async function setUserActive(db, id, isActive, adminId = null) {
+  // Convert string to ObjectId if needed
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+
+  let updateDoc;
+  if (isActive) {
+    // Enabling: clear disabledAt and disabledBy
+    updateDoc = {
+      $set: { isActive: true, disabledAt: null, disabledBy: null }
+    };
+  } else {
+    // Disabling: set disabledAt and disabledBy
+    // Convert string to ObjectId if needed, keep ObjectId as-is
+    const adminObjectId = adminId && typeof adminId === 'string'
+      ? new ObjectId(adminId)
+      : adminId;
+    updateDoc = {
+      $set: {
+        isActive: false,
+        disabledAt: new Date(),
+        disabledBy: adminObjectId
+      }
+    };
+  }
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { _id: objectId },
+    updateDoc,
+    { returnDocument: 'after' }
+  );
+
+  return result;
+}
+
+/**
+ * Create required indexes for the users collection
+ * Should be called once during application startup
+ *
+ * @param {Db} db - MongoDB database instance
+ * @returns {Promise<void>}
+ */
+async function createIndexes(db) {
+  const collection = db.collection(USER_COLLECTION);
+
+  // Unique index on googleId for OAuth lookups
+  await collection.createIndex(
+    { googleId: 1 },
+    { unique: true }
+  );
+
+  // Unique index on email for user identification
+  await collection.createIndex(
+    { email: 1 },
+    { unique: true }
+  );
+
+  // Compound index for querying active users sorted by last login
+  await collection.createIndex(
+    { isActive: 1, lastLoginAt: -1 }
+  );
+
+  // Index on role for admin queries
+  await collection.createIndex({ role: 1 });
+
+  // Compound index on isActive and role for filtered admin queries
+  await collection.createIndex({ isActive: 1, role: 1 });
+
+  // Index on createdAt for recent registrations
+  await collection.createIndex({ createdAt: -1 });
+
+  // Index on lastLoginAt for recently active users
+  await collection.createIndex({ lastLoginAt: -1 });
+
+  console.log('User collection indexes created successfully');
+}
+
+/**
+ * Get paginated list of users with optional filtering and sorting
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} options - Query options
+ * @param {number} options.page - Page number (1-indexed)
+ * @param {number} options.limit - Items per page (max 100)
+ * @param {string} options.search - Search term for name/email
+ * @param {string} options.status - Filter by status: 'all', 'active', 'disabled'
+ * @param {string} options.sort - Sort field: 'createdAt', 'lastLoginAt', 'loginCount'
+ * @param {string} options.order - Sort order: 'asc', 'desc'
+ * @returns {Promise<{users: Array, pagination: Object}>}
+ */
+async function getPaginatedUsers(db, options = {}) {
+  const page = Math.max(1, parseInt(options.page) || 1);
+  const limit = Math.min(100, Math.max(1, parseInt(options.limit) || 20));
+  const skip = (page - 1) * limit;
+
+  // Build filter
+  const filter = {};
+  if (options.search) {
+    const searchRegex = new RegExp(options.search, 'i');
+    filter.$or = [{ name: searchRegex }, { email: searchRegex }];
+  }
+  if (options.status === 'active') {
+    filter.isActive = true;
+  } else if (options.status === 'disabled') {
+    filter.isActive = false;
+  }
+
+  // Build sort
+  const sortField = ['createdAt', 'lastLoginAt', 'loginCount'].includes(options.sort)
+    ? options.sort
+    : 'createdAt';
+  const sortOrder = options.order === 'asc' ? 1 : -1;
+  const sort = { [sortField]: sortOrder };
+
+  const collection = db.collection(USER_COLLECTION);
+
+  // Execute queries in parallel
+  const [users, total] = await Promise.all([
+    collection.find(filter).sort(sort).skip(skip).limit(limit).toArray(),
+    collection.countDocuments(filter)
+  ]);
+
+  return {
+    users,
+    pagination: {
+      page,
+      limit,
+      total,
+      pages: Math.ceil(total / limit)
+    }
+  };
+}
+
+/**
+ * Update a user's role
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|ObjectId} id - User's MongoDB _id
+ * @param {string} role - New role ('user' or 'admin')
+ * @returns {Promise<Object|null>} Updated user or null
+ */
+async function updateUserRole(db, id, role) {
+  const objectId = typeof id === 'string' ? new ObjectId(id) : id;
+
+  const result = await db.collection(USER_COLLECTION).findOneAndUpdate(
+    { _id: objectId },
+    { $set: { role } },
+    { returnDocument: 'after' }
+  );
+
+  return result;
+}
+
+/**
+ * Check if a string is a valid MongoDB ObjectId
+ *
+ * @param {string} id - String to validate
+ * @returns {boolean} True if valid ObjectId format
+ */
+function isValidObjectId(id) {
+  if (typeof id !== 'string') return false;
+  if (!id || !id.trim()) return false;
+  return /^[0-9a-fA-F]{24}$/.test(id);
+}
+
+module.exports = {
+  USER_COLLECTION,
+  findOrCreateUser,
+  findByGoogleId,
+  findById,
+  setUserActive,
+  createIndexes,
+  getPaginatedUsers,
+  updateUserRole,
+  isValidObjectId
+};

package.json

@@ -4,28 +4,47 @@
     "description": "API backend for Country Club Towers & Gardens apartment dashboard",
     "main": "server.js",
     "scripts": {
-      "start": "node server.js",
-      "dev": "nodemon server.js",
-      "test": "echo \"Error: no test specified\" && exit 1"
+        "start": "node server.js",
+        "dev": "nodemon server.js",
+        "test": "jest",
+        "test:watch": "jest --watch",
+        "test:coverage": "jest --coverage",
+        "lint": "eslint .",
+        "lint:fix": "eslint . --fix"
     },
     "keywords": [
-      "apartments",
-      "api",
-      "real-estate",
-      "monitoring"
+        "apartments",
+        "api",
+        "real-estate",
+        "monitoring"
     ],
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
-      "express": "^4.18.2",
-      "mongodb": "^6.3.0",
-      "cors": "^2.8.5",
-      "express-rate-limit": "^7.1.5"
+        "axios": "^1.13.4",
+        "cheerio": "^1.2.0",
+        "cookie-parser": "^1.4.7",
+        "cors": "^2.8.5",
+        "cron-parser": "^5.5.0",
+        "express": "^4.18.2",
+        "express-rate-limit": "^7.1.5",
+        "jsonwebtoken": "^9.0.3",
+        "mongodb": "^6.3.0",
+        "node-cron": "^4.2.1",
+        "passport": "^0.7.0",
+        "passport-google-oauth20": "^2.0.0",
+        "uuid": "^13.0.0"
     },
     "devDependencies": {
-      "nodemon": "^3.0.2"
+        "@eslint/js": "^9.18.0",
+        "eslint": "^9.18.0",
+        "globals": "^15.14.0",
+        "jest": "^30.2.0",
+        "mongodb-memory-server": "^11.0.1",
+        "nodemon": "^3.0.2",
+        "supertest": "^7.2.2"
     },
     "engines": {
-      "node": ">=18.0.0"
+        "node": ">=18.0.0"
     }
-  }
+}

routes/activity.js

@@ -0,0 +1,179 @@
+const express = require('express');
+const { ObjectId } = require('mongodb');
+const { requireAuth, requireAdmin } = require('../middleware/auth');
+const { logActivity, ACTIONS, ACTIVITY_COLLECTION } = require('../services/activityLogger');
+
+const router = express.Router();
+
+/**
+ * POST /log - Log frontend activity
+ * Protected with requireAuth
+ */
+router.post('/log', requireAuth, async (req, res) => {
+  try {
+    const { action, metadata } = req.body;
+
+    // Validate action is in ACTIONS object
+    if (!action || !Object.values(ACTIONS).includes(action)) {
+      return res.status(400).json({ error: 'Invalid action type' });
+    }
+
+    const db = req.app.locals.db;
+
+    // Log the activity with merged metadata
+    await logActivity(
+      db,
+      req.user._id,
+      action,
+      { ...metadata, page: metadata?.page },
+      req
+    );
+
+    res.json({ success: true });
+  } catch (error) {
+    console.error('Error logging activity:', error);
+    res.status(500).json({ error: 'Failed to log activity' });
+  }
+});
+
+/**
+ * GET /user/:userId - Get activity for specific user
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/user/:userId', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const { userId } = req.params;
+    const limit = parseInt(req.query.limit) || 50;
+    const skip = parseInt(req.query.skip) || 0;
+
+    const db = req.app.locals.db;
+
+    // Convert userId to ObjectId
+    let userObjectId;
+    try {
+      userObjectId = new ObjectId(userId);
+    } catch (error) {
+      return res.status(400).json({ error: 'Invalid user ID format' });
+    }
+
+    // Find activities for the user
+    const activities = await db
+      .collection(ACTIVITY_COLLECTION)
+      .find({ userId: userObjectId })
+      .sort({ timestamp: -1 })
+      .skip(skip)
+      .limit(limit)
+      .toArray();
+
+    // Get total count
+    const total = await db
+      .collection(ACTIVITY_COLLECTION)
+      .countDocuments({ userId: userObjectId });
+
+    res.json({ activities, total });
+  } catch (error) {
+    console.error('Error fetching user activity:', error);
+    res.status(500).json({ error: 'Failed to fetch user activity' });
+  }
+});
+
+/**
+ * GET /recent - Get recent activity across all users
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/recent', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const limit = parseInt(req.query.limit) || 50;
+    const skip = parseInt(req.query.skip) || 0;
+
+    const db = req.app.locals.db;
+
+    // Find all activities sorted by timestamp
+    const activities = await db
+      .collection(ACTIVITY_COLLECTION)
+      .find({})
+      .sort({ timestamp: -1 })
+      .skip(skip)
+      .limit(limit)
+      .toArray();
+
+    // Get total count
+    const total = await db.collection(ACTIVITY_COLLECTION).countDocuments({});
+
+    res.json({ activities, total });
+  } catch (error) {
+    console.error('Error fetching recent activity:', error);
+    res.status(500).json({ error: 'Failed to fetch recent activity' });
+  }
+});
+
+/**
+ * GET /stats - Get activity statistics
+ * Protected with requireAuth and requireAdmin
+ */
+router.get('/stats', requireAuth, requireAdmin, async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+
+    // Calculate date 7 days ago
+    const sevenDaysAgo = new Date();
+    sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+
+    // Aggregate activity counts by action type in last 7 days
+    const actionCountsResult = await db
+      .collection(ACTIVITY_COLLECTION)
+      .aggregate([
+        {
+          $match: {
+            timestamp: { $gte: sevenDaysAgo }
+          }
+        },
+        {
+          $group: {
+            _id: '$action',
+            count: { $sum: 1 }
+          }
+        }
+      ])
+      .toArray();
+
+    // Convert array to object
+    const actionCounts = {};
+    actionCountsResult.forEach(item => {
+      actionCounts[item._id] = item.count;
+    });
+
+    // Count unique active users in last 7 days
+    const activeUsersResult = await db
+      .collection(ACTIVITY_COLLECTION)
+      .aggregate([
+        {
+          $match: {
+            timestamp: { $gte: sevenDaysAgo }
+          }
+        },
+        {
+          $group: {
+            _id: '$userId'
+          }
+        },
+        {
+          $count: 'total'
+        }
+      ])
+      .toArray();
+
+    const activeUsers = activeUsersResult.length > 0 ? activeUsersResult[0].total : 0;
+
+    res.json({
+      actionCounts,
+      activeUsers,
+      period: '7d'
+    });
+  } catch (error) {
+    console.error('Error fetching activity stats:', error);
+    res.status(500).json({ error: 'Failed to fetch activity statistics' });
+  }
+});
+
+module.exports = router;

routes/auth.js

@@ -0,0 +1,154 @@
+const express = require('express');
+const passport = require('passport');
+const jwt = require('jsonwebtoken');
+const crypto = require('crypto');
+const authConfig = require('../config/auth');
+const { requireAuth, generateToken } = require('../middleware/auth');
+const { logActivity, ACTIONS } = require('../services/activityLogger');
+
+const router = express.Router();
+
+/**
+ * GET /auth/google
+ * Initiates Google OAuth flow with state parameter for CSRF protection
+ */
+router.get('/google', (req, res, next) => {
+  // Generate cryptographically secure state parameter
+  const state = crypto.randomBytes(32).toString('hex');
+
+  // Store state in a short-lived cookie for validation
+  res.cookie('oauth_state', state, {
+    httpOnly: true,
+    secure: process.env.NODE_ENV === 'production',
+    sameSite: 'lax',
+    maxAge: 5 * 60 * 1000 // 5 minutes
+  });
+
+  passport.authenticate('google', {
+    scope: authConfig.google.scope,
+    session: false,
+    state: state
+  })(req, res, next);
+});
+
+/**
+ * GET /auth/google/callback
+ * Handles OAuth callback from Google
+ * Validates state parameter for CSRF protection
+ * On success: generates JWT, sets cookie, redirects to frontend
+ * On failure: redirects to login with error
+ */
+router.get('/google/callback', (req, res, next) => {
+  // Validate state parameter to prevent CSRF
+  const stateFromCookie = req.cookies.oauth_state;
+  const stateFromQuery = req.query.state;
+
+  // Clear the state cookie immediately
+  res.clearCookie('oauth_state');
+
+  if (!stateFromCookie || !stateFromQuery || stateFromCookie !== stateFromQuery) {
+    console.warn('OAuth state mismatch - potential CSRF attack');
+    return res.redirect(`${process.env.FRONTEND_URL}/login?error=invalid_state`);
+  }
+
+  // State is valid, proceed with authentication
+  passport.authenticate('google', {
+    session: false,
+    failureRedirect: `${process.env.FRONTEND_URL}/login?error=auth_failed`
+  })(req, res, next);
+}, async (req, res) => {
+    try {
+      const user = req.user;
+
+      // Check if email is verified (if available in profile)
+      if (req.authInfo && req.authInfo.emails && req.authInfo.emails[0]) {
+        const emailVerified = req.authInfo.emails[0].verified !== false; // Default to true if not present
+        if (!emailVerified) {
+          return res.redirect(`${process.env.FRONTEND_URL}/login?error=unverified`);
+        }
+      }
+
+      // Check if user account is active
+      if (!user.isActive) {
+        return res.redirect(`${process.env.FRONTEND_URL}/login`);
+      }
+
+      // Generate JWT token
+      const token = generateToken(user._id);
+
+      // Set auth cookie
+      res.cookie(authConfig.cookie.name, token, authConfig.cookie.options);
+
+      // Log login activity
+      const db = req.app.locals.db;
+      await logActivity(db, user._id, ACTIONS.LOGIN, { method: 'google' }, req);
+
+      // Redirect to frontend
+      res.redirect(process.env.FRONTEND_URL);
+    } catch (err) {
+      console.error('OAuth callback error:', err);
+      res.redirect(`${process.env.FRONTEND_URL}/login?error=auth_failed`);
+    }
+  }
+);
+
+/**
+ * GET /auth/me
+ * Returns current authenticated user's data
+ * Protected route - requires valid JWT
+ */
+router.get('/me', requireAuth, (req, res) => {
+  res.json({
+    user: {
+      id: req.user._id,
+      email: req.user.email,
+      name: req.user.name,
+      picture: req.user.picture,
+      role: req.user.role
+    }
+  });
+});
+
+/**
+ * POST /auth/logout
+ * Clears authentication cookie
+ * Protected route - requires valid JWT
+ */
+router.post('/logout', requireAuth, async (req, res) => {
+  // Log logout activity before clearing cookie
+  const db = req.app.locals.db;
+  await logActivity(db, req.user._id, ACTIONS.LOGOUT, {}, req);
+
+  // Clear the auth cookie
+  res.clearCookie(authConfig.cookie.name, {
+    ...authConfig.cookie.options,
+    maxAge: 0
+  });
+
+  res.json({ message: 'Logged out successfully' });
+});
+
+/**
+ * GET /auth/status
+ * Returns authentication status without requiring middleware
+ * Used for quick frontend checks
+ */
+router.get('/status', (req, res) => {
+  const token = req.cookies[authConfig.cookie.name];
+
+  // No token present
+  if (!token) {
+    return res.json({ authenticated: false });
+  }
+
+  try {
+    // Verify the token
+    jwt.verify(token, authConfig.jwt.secret);
+    return res.json({ authenticated: true });
+  } catch (err) {
+    // Token invalid or expired
+    return res.json({ authenticated: false });
+  }
+});
+
+module.exports = router;

server.js

@@ -2,12 +2,36 @@ const express = require('express');
 const { MongoClient } = require('mongodb');
 const cors = require('cors');
 const rateLimit = require('express-rate-limit');
+const cookieParser = require('cookie-parser');
+const passport = require('passport');
+const { configurePassport } = require('./middleware/passport');
+const { requireAuth } = require('./middleware/auth');
+const authRoutes = require('./routes/auth');
+const activityRoutes = require('./routes/activity');
+const adminRoutes = require('./routes/admin');
+const { createIndexes } = require('./models/user');
+const { createActivityIndexes } = require('./services/activityLogger');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
 
 // Configuration
-const MONGO_URI = process.env.MONGO_URI || "mongodb://admin:password123@localhost:27017";
+if (!process.env.MONGO_URI && process.env.NODE_ENV === 'production') {
+  console.error('❌ MONGO_URI environment variable is required in production');
+  process.exit(1);
+}
+const MONGO_URI = process.env.MONGO_URI || "mongodb://localhost:27017";
+
+// Log environment configuration status (safe - no secret values)
+console.log('📋 Environment Configuration:');
+console.log(`   NODE_ENV: ${process.env.NODE_ENV || 'development'}`);
+console.log(`   MONGO_URI: ${MONGO_URI ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CLIENT_ID: ${process.env.GOOGLE_CLIENT_ID ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CLIENT_SECRET: ${process.env.GOOGLE_CLIENT_SECRET ? '✓ Set' : '✗ Missing'}`);
+console.log(`   GOOGLE_CALLBACK_URL: ${process.env.GOOGLE_CALLBACK_URL || '✗ Missing'}`);
+console.log(`   JWT_SECRET: ${process.env.JWT_SECRET ? `✓ Set (${process.env.JWT_SECRET.length} chars)` : '✗ Missing'}`);
+console.log(`   FRONTEND_URL: ${process.env.FRONTEND_URL || 'http://localhost:5173 (default)'}`);
+console.log(`   ACTIVITY_LOG_LEVEL: ${process.env.ACTIVITY_LOG_LEVEL || 'all (default)'}`);
 const DB_NAME = "apartments";
 const UNITS_COLLECTION = "units_migration_test";
 const PRICES_COLLECTION = "unit_prices_migration_test";
@@ -16,7 +40,17 @@ const DAILY_SUMMARIES_COLLECTION = "daily_summaries";
 console.log(`🔗 Connecting to MongoDB: ${MONGO_URI.includes('localhost') ? 'Local MongoDB' : 'Atlas MongoDB'}`);
 
 // Middleware
-app.use(cors());
+const FRONTEND_URL = process.env.FRONTEND_URL || 'http://localhost:5173';
+
+const corsOptions = {
+  origin: FRONTEND_URL,
+  credentials: true,
+  methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
+  allowedHeaders: ['Content-Type', 'Authorization'],
+  exposedHeaders: ['set-cookie']
+};
+app.use(cors(corsOptions));
+app.use(cookieParser());
 app.use(express.json());
 
 // Rate limiting
@@ -26,6 +60,9 @@ const limiter = rateLimit({
 });
 app.use(limiter);
 
+// Initialize Passport
+app.use(passport.initialize());
+
 // MongoDB connection
 let db;
 let client;
@@ -35,23 +72,84 @@ async function connectToMongoDB() {
     client = new MongoClient(MONGO_URI);
     await client.connect();
     db = client.db(DB_NAME);
+    app.locals.db = db;
     console.log('✅ Successfully connected to MongoDB');
+
+    // Configure Passport and create indexes after DB connection
+    configurePassport(passport, db);
+    await createIndexes(db);
+    await createActivityIndexes(db);
+    console.log('✅ Passport configured and indexes created');
   } catch (error) {
     console.error('❌ Failed to connect to MongoDB:', error);
     process.exit(1);
   }
 }
 
-// Helper function to get today's date
-const getTodayDate = () => new Date().toISOString().split('T')[0];
+// Helper function to get today's date in UTC
+const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
 
-// Helper function to get yesterday's date
-const getYesterdayDate = () => {
-  const yesterday = new Date();
-  yesterday.setDate(yesterday.getDate() - 1);
-  return yesterday.toISOString().split('T')[0];
+// Cache for the most recent date with data
+let cachedLatestDate = null;
+let cachedLatestDateTimestamp = 0;
+const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+
+// Helper function to get the most recent date with data in the database
+// This handles timezone mismatches between server and data collection
+const getLatestDateWithData = async () => {
+  const now = Date.now();
+
+  // Return cached value if still valid
+  if (cachedLatestDate && (now - cachedLatestDateTimestamp) < CACHE_TTL) {
+    return cachedLatestDate;
+  }
+
+  try {
+    const result = await db.collection(PRICES_COLLECTION)
+      .find({})
+      .sort({ date_checked: -1 })
+      .limit(1)
+      .project({ date_checked: 1 })
+      .toArray();
+
+    if (result.length > 0) {
+      cachedLatestDate = result[0].date_checked;
+      cachedLatestDateTimestamp = now;
+      return cachedLatestDate;
+    }
+  } catch (error) {
+    console.error('Error fetching latest date with data:', error);
+  }
+
+  // Fallback to UTC today if no data found
+  return getTodayDateUTC();
 };
 
+// Helper function to get yesterday relative to the latest date with data
+const getYesterdayDate = (today) => {
+  const todayDate = new Date(today + 'T00:00:00Z');
+  todayDate.setDate(todayDate.getDate() - 1);
+  return todayDate.toISOString().split('T')[0];
+};
+
+// Helper function to validate unit code (prevents NoSQL injection)
+const isValidUnitCode = (unitCode) => {
+  // Allow alphanumeric characters, hyphens, and underscores, max 20 chars
+  return typeof unitCode === 'string' &&
+         unitCode.length > 0 &&
+         unitCode.length <= 20 &&
+         /^[A-Za-z0-9_-]+$/.test(unitCode);
+};
+
+// Mount auth routes
+app.use('/auth', authRoutes);
+
+// Mount activity routes
+app.use('/activity', activityRoutes);
+
+// Mount admin routes (Traefik strips /api prefix, so /api/admin becomes /admin)
+app.use('/admin', adminRoutes);
+
 // Health check endpoint
 app.get('/health', (req, res) => {
   res.json({
@@ -62,7 +160,7 @@ app.get('/health', (req, res) => {
 });
 
 // Get last scrape time
-app.get('/last-scrape', async (req, res) => {
+app.get('/last-scrape', requireAuth, async (req, res) => {
   try {
     // Get the most recent price record using _id (ObjectId contains timestamp)
     const lastRecord = await db.collection(PRICES_COLLECTION)
@@ -94,9 +192,9 @@ app.get('/last-scrape', async (req, res) => {
 });
 
 // Get daily summary (matches your daily_summaries collection)
-app.get('/daily-summary', async (req, res) => {
+app.get('/daily-summary', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     
     // Try to get today's summary first
     let summary = await db.collection(DAILY_SUMMARIES_COLLECTION)
@@ -133,14 +231,16 @@ app.get('/daily-summary', async (req, res) => {
 });
 
 // Get price history (last 15 days of average prices)
-app.get('/price-history', async (req, res) => {
+app.get('/price-history', requireAuth, async (req, res) => {
   try {
     const days = parseInt(req.query.days) || 15;
+    const latestDate = await getLatestDateWithData();
 
-    // Generate date range
+    // Generate date range ending at the latest date with data
     const dates = [];
+    const baseDate = new Date(latestDate + 'T00:00:00Z');
     for (let i = days - 1; i >= 0; i--) {
-      const date = new Date();
+      const date = new Date(baseDate);
       date.setDate(date.getDate() - i);
       dates.push(date.toISOString().split('T')[0]);
     }
@@ -188,9 +288,10 @@ app.get('/price-history', async (req, res) => {
 });
 
 // Get available units with current prices
-app.get('/available-units', async (req, res) => {
+app.get('/available-units', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
+    const yesterday = getYesterdayDate(today);
 
     // Get units that have prices today (excluding furnished units)
     const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
@@ -215,11 +316,17 @@ app.get('/available-units', async (req, res) => {
       },
       {
         $addFields: {
-          currentPrice: {
+          currentPriceDoc: {
             $arrayElemAt: [
               { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", today] } } },
               0
             ]
+          },
+          yesterdayPriceDoc: {
+            $arrayElemAt: [
+              { $filter: { input: "$price_info", cond: { $eq: ["$$this.date_checked", yesterday] } } },
+              0
+            ]
           }
         }
       },
@@ -273,26 +380,52 @@ app.get('/available-units', async (req, res) => {
                 }
               }
             }
+          }
+        }
+      },
+      {
+        $addFields: {
+          // Filter prices to only include those from availability start date onwards
+          availabilityPrices: {
+            $filter: {
+              input: "$all_prices",
+              cond: { $gte: ["$$this.date_checked", "$availabilityCalc.startDate"] }
+            }
           },
+          priceHistory: {
+            $slice: [
+              { $sortArray: { input: "$all_prices", sortBy: { date_checked: -1 } } },
+              30
+            ]
+          }
+        }
+      },
+      {
+        $addFields: {
           priceStats: {
             $let: {
               vars: {
-                prices: { $map: { input: "$all_prices", as: "p", in: "$$p.price" } }
+                prices: { $map: { input: "$availabilityPrices", as: "p", in: "$$p.price" } }
               },
               in: {
                 minPrice: { $min: "$$prices" },
-                maxPrice: { $max: "$$prices" },
-                priceHistory: {
-                  $slice: [
-                    { $sortArray: { input: "$all_prices", sortBy: { date_checked: -1 } } },
-                    30
-                  ]
-                }
+                maxPrice: { $max: "$$prices" }
               }
             }
           }
         }
       },
+      {
+        $addFields: {
+          priceChangeToday: {
+            $cond: {
+              if: { $and: [{ $ne: ["$currentPriceDoc.price", null] }, { $ne: ["$yesterdayPriceDoc.price", null] }] },
+              then: { $subtract: ["$currentPriceDoc.price", "$yesterdayPriceDoc.price"] },
+              else: null
+            }
+          }
+        }
+      },
       {
         $project: {
           unitCode: "$unit_code",
@@ -303,7 +436,9 @@ app.get('/available-units', async (req, res) => {
           community: "$community",
           available: "$available",
           soonest: "$soonest",
-          currentPrice: "$currentPrice.price",
+          currentPrice: "$currentPriceDoc.price",
+          yesterdayPrice: "$yesterdayPriceDoc.price",
+          priceChangeToday: { $ifNull: ["$priceChangeToday", null] },
           minPrice: "$priceStats.minPrice",
           maxPrice: "$priceStats.maxPrice",
           daysAvailable: {
@@ -315,7 +450,7 @@ app.get('/available-units', async (req, res) => {
           },
           priceHistory: {
             $map: {
-              input: "$priceStats.priceHistory",
+              input: "$priceHistory",
               as: "ph",
               in: {
                 date: "$$ph.date_checked",
@@ -336,10 +471,15 @@ app.get('/available-units', async (req, res) => {
 });
 
 // Get full price history for a specific unit
-app.get('/unit/:unitCode/price-history', async (req, res) => {
+app.get('/unit/:unitCode/price-history', requireAuth, async (req, res) => {
   try {
     const { unitCode } = req.params;
 
+    // Validate unitCode to prevent NoSQL injection
+    if (!isValidUnitCode(unitCode)) {
+      return res.status(400).json({ error: 'Invalid unit code format' });
+    }
+
     const priceHistory = await db.collection(PRICES_COLLECTION)
       .find({ unit_code: unitCode })
       .sort({ date_checked: 1 })
@@ -363,9 +503,9 @@ app.get('/unit/:unitCode/price-history', async (req, res) => {
 });
 
 // Get comprehensive analytics data
-app.get('/analytics', async (req, res) => {
+app.get('/analytics', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
 
     // 1. Monthly Price Trends - aggregate all prices by month
     const monthlyTrends = await db.collection(PRICES_COLLECTION).aggregate([
@@ -605,10 +745,10 @@ app.get('/analytics', async (req, res) => {
 });
 
 // Get recent activity (new units, rented units, price changes)
-app.get('/recent-activity', async (req, res) => {
+app.get('/recent-activity', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
-    const yesterday = getYesterdayDate();
+    const today = await getLatestDateWithData();
+    const yesterday = getYesterdayDate(today);
     
     // Get today's summary for new/rented units
     const todaySummary = await db.collection(DAILY_SUMMARIES_COLLECTION)
@@ -734,9 +874,9 @@ app.get('/recent-activity', async (req, res) => {
 });
 
 // Get plan statistics
-app.get('/plan-stats', async (req, res) => {
+app.get('/plan-stats', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     
     const planStats = await db.collection(UNITS_COLLECTION).aggregate([
       {
@@ -797,9 +937,9 @@ app.get('/plan-stats', async (req, res) => {
 
 // Expanded api
 // Get best deals (units priced below their historical average)
-app.get('/best-deals', async (req, res) => {
+app.get('/best-deals', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const limit = parseInt(req.query.limit) || 5;
 
     const bestDeals = await db.collection(UNITS_COLLECTION).aggregate([
@@ -922,16 +1062,17 @@ app.get('/best-deals', async (req, res) => {
 });
 
 // Get price drops (units with recent price decreases)
-app.get('/price-drops', async (req, res) => {
+app.get('/price-drops', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const daysBack = parseInt(req.query.days) || 7;
     const limit = parseInt(req.query.limit) || 10;
 
-    // Generate date range for the last N days
+    // Generate date range for the last N days relative to latest data date
     const dates = [];
+    const baseDate = new Date(today + 'T00:00:00Z');
     for (let i = 0; i < daysBack; i++) {
-      const date = new Date();
+      const date = new Date(baseDate);
       date.setDate(date.getDate() - i);
       dates.push(date.toISOString().split('T')[0]);
     }
@@ -1019,9 +1160,9 @@ app.get('/price-drops', async (req, res) => {
 });
 
 // Get stale inventory (units on market for a long time)
-app.get('/stale-inventory', async (req, res) => {
+app.get('/stale-inventory', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
     const minDays = parseInt(req.query.minDays) || 10;
     const limit = parseInt(req.query.limit) || 10;
 
@@ -1145,9 +1286,9 @@ app.get('/stale-inventory', async (req, res) => {
 });
 
 // Get market insights and predictions
-app.get('/market-insights', async (req, res) => {
+app.get('/market-insights', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
+    const today = await getLatestDateWithData();
 
     // Get various market metrics
     const [
@@ -1355,10 +1496,10 @@ app.get('/market-insights', async (req, res) => {
 });
 
 // Enhanced available units with more details
-app.get('/available-units-enhanced', async (req, res) => {
+app.get('/available-units-enhanced', requireAuth, async (req, res) => {
   try {
-    const today = getTodayDate();
-    const yesterday = getYesterdayDate();
+    const today = await getLatestDateWithData();
+    const yesterday = getYesterdayDate(today);
 
     const availableUnits = await db.collection(UNITS_COLLECTION).aggregate([
       {
@@ -1541,10 +1682,16 @@ app.get('/available-units-enhanced', async (req, res) => {
 });
 
 // Get unit details by unit code
-app.get('/unit/:unitCode', async (req, res) => {
+app.get('/unit/:unitCode', requireAuth, async (req, res) => {
   try {
     const { unitCode } = req.params;
-    const today = getTodayDate();
+
+    // Validate unitCode to prevent NoSQL injection
+    if (!isValidUnitCode(unitCode)) {
+      return res.status(400).json({ error: 'Invalid unit code format' });
+    }
+
+    const today = await getLatestDateWithData();
 
     const unit = await db.collection(UNITS_COLLECTION).aggregate([
       { $match: { unit_code: unitCode } },

services/activityLogger.js

@@ -0,0 +1,192 @@
+const { ObjectId } = require('mongodb');
+
+// Collection name
+const ACTIVITY_COLLECTION = 'user_activity';
+
+// Activity action constants
+const ACTIONS = {
+  LOGIN: 'login',
+  LOGOUT: 'logout',
+  PAGE_VIEW: 'page_view',
+  NAVIGATION: 'navigation',
+  UNIT_VIEW: 'unit_view',
+  UNIT_WATCH: 'unit_watch',
+  FILTER_CHANGE: 'filter_change',
+  SORT_CHANGE: 'sort_change',
+  SEARCH: 'search',
+  EXPORT: 'export',
+  VIEW_TOGGLE: 'view_toggle'
+};
+
+// Log levels define which actions should be logged
+const LOG_LEVELS = {
+  all: [
+    'login',
+    'logout',
+    'page_view',
+    'navigation',
+    'filter_change',
+    'sort_change',
+    'search',
+    'unit_view',
+    'unit_watch',
+    'export',
+    'view_toggle'
+  ],
+  navigation: [
+    'login',
+    'logout',
+    'page_view',
+    'navigation'
+  ],
+  none: []
+};
+
+/**
+ * Get the current activity log level from environment variable
+ * @returns {string} Current log level ('all', 'navigation', or 'none')
+ */
+function getActivityLevel() {
+  const level = process.env.ACTIVITY_LOG_LEVEL || 'all';
+
+  // Validate level exists, default to 'all' if invalid
+  if (!LOG_LEVELS[level]) {
+    console.warn(`Invalid ACTIVITY_LOG_LEVEL: ${level}, defaulting to 'all'`);
+    return 'all';
+  }
+
+  return level;
+}
+
+/**
+ * Check if an action should be logged based on current log level
+ * @param {string} action - Action to check
+ * @returns {boolean} True if action should be logged
+ */
+function shouldLog(action) {
+  const level = getActivityLevel();
+  const allowedActions = LOG_LEVELS[level];
+  return allowedActions.includes(action);
+}
+
+/**
+ * Check if an action is an admin action (always logged regardless of log level)
+ * @param {string} action - Action to check
+ * @returns {boolean} True if admin action
+ */
+function isAdminAction(action) {
+  return action && action.startsWith('ADMIN_');
+}
+
+/**
+ * Log a user activity to the database
+ * Supports two calling conventions:
+ * 1. logActivity(db, userId, action, metadata, req) - positional parameters
+ * 2. logActivity(db, { userId, action, metadata }) - object parameter for admin actions
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {string|Object} userIdOrOptions - User ID string or options object
+ * @param {string} [action] - Action type (use ACTIONS constants)
+ * @param {Object} [metadata] - Additional action-specific data
+ * @param {Object} [req] - Express request object (for IP and user agent)
+ * @returns {Promise<void>}
+ */
+async function logActivity(db, userIdOrOptions, action, metadata = {}, req = null) {
+  let userId;
+  let actualAction;
+  let actualMetadata;
+  let actualReq;
+
+  // Support object-based call for admin actions
+  if (typeof userIdOrOptions === 'object' && userIdOrOptions !== null) {
+    userId = userIdOrOptions.userId;
+    actualAction = userIdOrOptions.action;
+    actualMetadata = userIdOrOptions.metadata || {};
+    actualReq = userIdOrOptions.req || null;
+  } else {
+    userId = userIdOrOptions;
+    actualAction = action;
+    actualMetadata = metadata;
+    actualReq = req;
+  }
+
+  // Admin actions bypass log level filtering
+  if (!isAdminAction(actualAction) && !shouldLog(actualAction)) {
+    return;
+  }
+
+  try {
+    // Extract IP address (handle proxy forwarding)
+    let ip = null;
+    if (actualReq) {
+      ip = actualReq.ip || actualReq.headers?.['x-forwarded-for']?.split(',')[0] || null;
+    }
+
+    // Extract user agent
+    const userAgent = actualReq?.headers?.['user-agent'] || null;
+
+    // Create activity document
+    const activityDoc = {
+      userId: new ObjectId(userId),
+      action: actualAction,
+      metadata: actualMetadata || {},
+      page: actualMetadata?.page || null,
+      timestamp: new Date(),
+      userAgent: userAgent,
+      ip: ip
+    };
+
+    // Insert into user_activity collection
+    await db.collection(ACTIVITY_COLLECTION).insertOne(activityDoc);
+  } catch (error) {
+    // Log error but don't throw - activity logging should not break the main flow
+    console.error('Error logging activity:', error);
+  }
+}
+
+/**
+ * Create required indexes for the user_activity collection
+ * @param {Db} db - MongoDB database instance
+ * @returns {Promise<void>}
+ */
+async function createActivityIndexes(db) {
+  try {
+    const collection = db.collection(ACTIVITY_COLLECTION);
+
+    // Index for user activity queries (get all activities for a user, sorted by time)
+    await collection.createIndex(
+      { userId: 1, timestamp: -1 },
+      { name: 'userId_timestamp' }
+    );
+
+    // Index for action type queries (get all activities of a certain type)
+    await collection.createIndex(
+      { action: 1, timestamp: -1 },
+      { name: 'action_timestamp' }
+    );
+
+    // TTL index to automatically delete activities older than 90 days
+    await collection.createIndex(
+      { timestamp: 1 },
+      {
+        name: 'timestamp_ttl',
+        expireAfterSeconds: 7776000  // 90 days = 90 * 24 * 60 * 60
+      }
+    );
+
+    console.log('Activity collection indexes created successfully');
+  } catch (error) {
+    console.error('Error creating activity indexes:', error);
+    throw error;
+  }
+}
+
+module.exports = {
+  ACTIVITY_COLLECTION,
+  ACTIONS,
+  LOG_LEVELS,
+  getActivityLevel,
+  shouldLog,
+  logActivity,
+  createActivityIndexes
+};

services/scraperLogger.js

@@ -0,0 +1,164 @@
+/**
+ * Scraper-specific structured JSON logger
+ * Produces one JSON object per line to stdout
+ */
+
+const LEVELS = {
+  info: 'info',
+  warn: 'warn',
+  error: 'error'
+};
+
+/**
+ * Create a logger instance scoped to a job ID
+ * @param {string} jobId - Job identifier for correlation
+ * @returns {Object} Logger object with info, warn, error methods
+ */
+function createLogger(jobId) {
+  /**
+   * Internal log function
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} context - Additional context data
+   */
+  const log = (level, message, context = {}) => {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      message: truncateMessage(message),
+      jobId,
+      context: sanitizeContext(context)
+    };
+
+    // Output as single-line JSON
+    console.log(JSON.stringify(entry));
+  };
+
+  return {
+    info: (message, context) => log(LEVELS.info, message, context),
+    warn: (message, context) => log(LEVELS.warn, message, context),
+    error: (message, context) => log(LEVELS.error, message, context)
+  };
+}
+
+/**
+ * Truncate message to prevent log bloat
+ * @param {string} message - Message to truncate
+ * @param {number} maxLength - Maximum length (default 1000)
+ * @returns {string} Truncated message
+ */
+function truncateMessage(message, maxLength = 1000) {
+  if (message === null || message === undefined) {
+    return '';
+  }
+  const str = String(message);
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.substring(0, maxLength) + '... [truncated]';
+}
+
+/**
+ * Recursively process an object to handle Buffers, circular references, and sensitive data
+ * @param {*} obj - Object to process
+ * @param {WeakSet} seen - Set of seen objects for circular reference detection
+ * @returns {*} Processed value
+ */
+function processValue(obj, seen = new WeakSet()) {
+  // Handle null/undefined
+  if (obj === null) {
+    return null;
+  }
+  if (obj === undefined) {
+    return null;
+  }
+
+  // Handle Buffer BEFORE checking for object (Buffer is an object)
+  if (Buffer.isBuffer(obj)) {
+    return `[Buffer: ${obj.length} bytes]`;
+  }
+
+  // Handle strings - check for MongoDB connection strings
+  if (typeof obj === 'string') {
+    if (/mongodb(\+srv)?:\/\//.test(obj)) {
+      return redactConnectionString(obj);
+    }
+    return obj;
+  }
+
+  // Handle primitives
+  if (typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Handle circular references
+  if (seen.has(obj)) {
+    return '[Circular]';
+  }
+  seen.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => processValue(item, seen));
+  }
+
+  // Handle plain objects
+  const result = {};
+  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
+
+  for (const key of Object.keys(obj)) {
+    // Check for sensitive keys
+    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
+      result[key] = '[REDACTED]';
+    } else {
+      result[key] = processValue(obj[key], seen);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sanitize context object for safe logging
+ * - Remove circular references
+ * - Redact sensitive data
+ * - Handle special types (Buffer, undefined)
+ * @param {Object} context - Context object
+ * @returns {Object} Sanitized context
+ */
+function sanitizeContext(context) {
+  if (!context || typeof context !== 'object') {
+    return {};
+  }
+
+  try {
+    return processValue(context);
+  } catch (error) {
+    // If sanitization fails, return empty context
+    return { sanitizationError: 'Failed to sanitize context' };
+  }
+}
+
+/**
+ * Redact credentials from MongoDB connection string
+ * @param {string} uri - Connection string
+ * @returns {string} Redacted string
+ */
+function redactConnectionString(uri) {
+  try {
+    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
+    return uri.replace(
+      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
+      'mongodb$1://[user]:[REDACTED]@'
+    );
+  } catch {
+    return '[REDACTED CONNECTION STRING]';
+  }
+}
+
+module.exports = {
+  createLogger,
+  LEVELS,
+  // Export internal functions for testing
+  redactConnectionString
+};

services/scraperService.js

@@ -0,0 +1,838 @@
+/**
+ * Scraper Service
+ *
+ * Core scraper logic including HTTP fetching, HTML parsing,
+ * data transformation, and database operations.
+ */
+
+const axios = require('axios');
+const cheerio = require('cheerio');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { createLogger } = require('./scraperLogger');
+
+/**
+ * Sleep utility for retry delays
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Determine if an error is retryable
+ * @param {Error} error - Axios error
+ * @returns {boolean} True if should retry
+ */
+function isRetryableError(error) {
+  // Network errors (timeout, DNS, connection) don't have a response property
+  if (!error.response) {
+    return true;
+  }
+
+  const status = error.response.status;
+
+  // 5xx server errors are retryable
+  if (status >= 500) {
+    return true;
+  }
+
+  // 429 Too Many Requests - do not retry immediately
+  if (status === 429) {
+    return false;
+  }
+
+  // 4xx client errors - do not retry
+  return false;
+}
+
+/**
+ * Fetch page HTML with retry logic
+ * @param {string} url - Target URL
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<string>} HTML content
+ * @throws {Error} After all retries exhausted
+ */
+async function fetchPage(url, logger) {
+  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    try {
+      logger.info('Fetching page', { url, attempt });
+
+      const response = await axios.get(url, {
+        timeout,
+        headers: {
+          'User-Agent': config.USER_AGENT
+        },
+        maxRedirects: 5,
+        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
+      });
+
+      logger.info('Page fetched successfully', {
+        status: response.status,
+        contentLength: response.data.length
+      });
+
+      return response.data;
+
+    } catch (error) {
+      lastError = error;
+
+      // Determine if error is retryable
+      const isRetryable = isRetryableError(error);
+
+      logger.warn('Fetch attempt failed', {
+        attempt,
+        errorType: error.name,
+        errorMessage: error.message,
+        statusCode: error.response?.status,
+        isRetryable
+      });
+
+      // Don't retry non-retryable errors (4xx)
+      if (!isRetryable) {
+        throw error;
+      }
+
+      // Don't wait after last attempt
+      if (attempt <= maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+        logger.info('Waiting before retry', { delay });
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Parse unit data from HTML using cheerio
+ * @param {string} html - HTML content
+ * @param {Object} logger - Logger instance
+ * @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
+ */
+function parseUnits(html, logger) {
+  const $ = cheerio.load(html);
+  const units = [];
+
+  // Find the main container
+  const container = $('section.spaces__tab-unit');
+
+  if (container.length === 0) {
+    logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
+    return [];
+  }
+
+  // Extract each article element
+  container.find('article').each((index, article) => {
+    const $article = $(article);
+
+    const unit = {
+      // Core identifiers
+      id: $article.attr('data-spaces-id'),
+      unit_code: $article.attr('data-spaces-unit'),
+      unit_id: $article.attr('data-spaces-unit-id'),
+
+      // Physical attributes
+      floor: $article.attr('data-spaces-unit-floor'),
+      area: $article.attr('data-spaces-sort-area'),
+      bed_count: $article.attr('data-spaces-sort-bed'),
+      bath_count: $article.attr('data-spaces-sort-bath'),
+
+      // Pricing
+      price: $article.attr('data-spaces-sort-price'),
+
+      // Availability
+      available: $article.attr('data-spaces-available'),
+      unavailable: $article.attr('data-spaces-unavailable'),
+      soonest: $article.attr('data-spaces-soonest'),
+      date_available: $article.attr('data-spaces-sort-date'),
+
+      // Plan information
+      plan_id: $article.attr('data-spaces-plan-id'),
+      plan_name: $article.attr('data-spaces-sort-plan-name'),
+
+      // Property information
+      obj_type: $article.attr('data-spaces-obj'),
+      community: $article.attr('data-spaces-community'),
+      asset: $article.attr('data-spaces-asset'),
+
+      // URLs
+      href: $article.attr('data-spaces-href'),
+      inventory_href: $article.attr('data-spaces-inventory-href'),
+
+      // Specials
+      specials_content: $article.attr('data-spaces-specials-content')
+    };
+
+    // Extract image URL from nested element if present
+    const imgElement = $article.find('img').first();
+    if (imgElement.length > 0) {
+      unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
+    }
+
+    units.push(unit);
+  });
+
+  logger.info('Units parsed', { count: units.length });
+
+  // Deduplicate by unit_code (in case of duplicate articles)
+  const seen = new Set();
+  const deduplicated = units.filter(unit => {
+    if (!unit.unit_code || seen.has(unit.unit_code)) {
+      return false;
+    }
+    seen.add(unit.unit_code);
+    return true;
+  });
+
+  if (deduplicated.length < units.length) {
+    logger.warn('Duplicate units removed', {
+      original: units.length,
+      deduplicated: deduplicated.length
+    });
+  }
+
+  return deduplicated;
+}
+
+// ============================================================
+// Data Type Conversion Helpers
+// ============================================================
+
+/**
+ * Parse string to integer, return null for invalid
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed integer or null
+ */
+function parseInteger(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Parse string to positive integer, return null for 0 or invalid.
+ * Used for fields like area where 0 means "not available".
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed positive integer or null
+ */
+function parsePositiveInteger(value) {
+  const parsed = parseInteger(value);
+  return parsed === 0 ? null : parsed;
+}
+
+/**
+ * Parse value that could be integer or string identifier.
+ * Returns integer if cleanly parseable, otherwise trimmed string.
+ * @param {*} value - Value to parse
+ * @returns {number|string|null} Parsed integer, trimmed string, or null
+ */
+function parseIntegerOrString(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  // If it parses cleanly to an integer, return integer
+  if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
+    return parsed;
+  }
+  // Otherwise return as trimmed string
+  return String(value).trim();
+}
+
+/**
+ * Parse price, stripping non-numeric characters.
+ * Handles "$1,234" format and "Call for pricing" text.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed price or null
+ */
+function parsePrice(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+
+  // Check for "Call for pricing" or similar text
+  if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
+    return null;
+  }
+
+  // Strip non-numeric characters except decimal point
+  const cleaned = String(value).replace(/[^0-9.]/g, '');
+  const parsed = parseInt(cleaned, 10);
+
+  if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
+    return null;
+  }
+
+  // Negative prices are invalid
+  if (parsed < 0) {
+    return null;
+  }
+
+  return parsed;
+}
+
+/**
+ * Parse boolean from string.
+ * Handles "true"/"false"/"1"/"0" and actual boolean values.
+ * @param {*} value - Value to parse
+ * @returns {boolean|null} Parsed boolean or null
+ */
+function parseBoolean(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  const str = String(value).toLowerCase().trim();
+  if (str === 'true' || str === '1') return true;
+  if (str === 'false' || str === '0') return false;
+  return null;
+}
+
+/**
+ * Parse float value.
+ * Named parseFloatValue to avoid shadowing the global parseFloat.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed float or null
+ */
+function parseFloatValue(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Trim string, return null for empty.
+ * Converts non-string values to string before trimming.
+ * @param {*} value - Value to trim
+ * @returns {string|null} Trimmed string or null
+ */
+function trimString(value) {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  const trimmed = String(value).trim();
+  return trimmed === '' ? null : trimmed;
+}
+
+// ============================================================
+// Main Data Type Conversion Function
+// ============================================================
+
+/**
+ * Convert unit data types from strings to proper types.
+ * Applies the appropriate parser to each field based on its expected type.
+ * @param {Object} unit - Raw unit object with string values
+ * @returns {Object} Unit object with converted types
+ */
+function convertDataTypes(unit) {
+  return {
+    // Integer fields
+    id: parseInteger(unit.id),
+    unit_id: parseIntegerOrString(unit.unit_id),
+    floor: parseInteger(unit.floor),
+    area: parsePositiveInteger(unit.area),
+    bed_count: parseInteger(unit.bed_count),
+    plan_id: parseInteger(unit.plan_id),
+    asset: parseInteger(unit.asset),
+    date_available: parseInteger(unit.date_available),
+
+    // Float fields
+    bath_count: parseFloatValue(unit.bath_count),
+
+    // Price field (special handling)
+    price: parsePrice(unit.price),
+
+    // Boolean fields
+    available: parseBoolean(unit.available),
+    unavailable: parseBoolean(unit.unavailable),
+
+    // String fields (trim and preserve)
+    unit_code: trimString(unit.unit_code),
+    plan_name: trimString(unit.plan_name),
+    soonest: trimString(unit.soonest),
+    obj_type: trimString(unit.obj_type),
+    community: trimString(unit.community),
+    href: trimString(unit.href),
+    inventory_href: trimString(unit.inventory_href),
+    image_url: trimString(unit.image_url),
+    specials_content: trimString(unit.specials_content)
+  };
+}
+
+// ============================================================
+// Date Helpers
+// ============================================================
+
+/**
+ * Calculate yesterday's date from a given date string.
+ * @param {string} dateStr - Date in YYYY-MM-DD format
+ * @returns {string} Yesterday's date in YYYY-MM-DD format
+ */
+function getYesterday(dateStr) {
+  const date = new Date(dateStr + 'T00:00:00Z');
+  date.setUTCDate(date.getUTCDate() - 1);
+  return date.toISOString().split('T')[0];
+}
+
+// ============================================================
+// Database Operations
+// ============================================================
+
+/**
+ * Upsert unit records to database using bulkWrite.
+ * Each unit is matched by unit_code as the unique key.
+ * Sets last_scraped and data_source on every update.
+ * Sets first_seen only on initial insert via $setOnInsert.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Bulk write result
+ */
+async function upsertUnits(db, units, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+  const now = new Date().toISOString();
+
+  const operations = units.map(unit => ({
+    updateOne: {
+      filter: { unit_code: unit.unit_code },
+      update: {
+        $set: {
+          ...unit,
+          last_scraped: now,
+          data_source: 'web_scraper'
+        },
+        $setOnInsert: {
+          first_seen: now
+        }
+      },
+      upsert: true
+    }
+  }));
+
+  if (operations.length === 0) {
+    logger.warn('No units to upsert');
+    return { modifiedCount: 0, upsertedCount: 0 };
+  }
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Units upserted', {
+      matched: result.matchedCount,
+      modified: result.modifiedCount,
+      upserted: result.upsertedCount
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to upsert units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Insert price records for today using bulkWrite with upsert.
+ * Uses unit_code + date_checked as the composite key to prevent
+ * duplicate price records for the same unit on the same day.
+ * Re-running on the same day updates existing records (idempotent).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {string} date - Date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Insert result with insertedCount
+ */
+async function insertPrices(db, units, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+  const now = new Date().toISOString();
+
+  // Filter out units without prices
+  const unitsWithPrices = units.filter(u => u.price !== null);
+
+  const priceRecords = unitsWithPrices.map(unit => ({
+    unit_code: unit.unit_code,
+    date_checked: date,
+    price: unit.price,
+    last_updated: now,
+    data_source: 'web_scraper'
+  }));
+
+  if (priceRecords.length === 0) {
+    logger.warn('No price records to insert');
+    return { insertedCount: 0 };
+  }
+
+  // Use updateOne with upsert to handle re-runs on same day
+  const operations = priceRecords.map(record => ({
+    updateOne: {
+      filter: {
+        unit_code: record.unit_code,
+        date_checked: record.date_checked
+      },
+      update: { $set: record },
+      upsert: true
+    }
+  }));
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Prices inserted', {
+      inserted: result.upsertedCount,
+      updated: result.modifiedCount
+    });
+
+    return { insertedCount: result.upsertedCount + result.modifiedCount };
+
+  } catch (error) {
+    logger.error('Failed to insert prices', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Mark units not in current scrape as stale/unavailable.
+ * Uses updateMany to set available: false and marked_stale_date
+ * for all units whose unit_code is NOT in the current scrape
+ * and that are currently available.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Set<string>} currentUnitCodes - Unit codes from current scrape
+ * @param {string} date - Current date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Update result
+ */
+async function markStaleUnits(db, currentUnitCodes, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+
+  try {
+    const result = await collection.updateMany(
+      {
+        unit_code: { $nin: [...currentUnitCodes] },
+        available: true
+      },
+      {
+        $set: {
+          available: false,
+          marked_stale_date: date
+        }
+      }
+    );
+
+    logger.info('Stale units marked', { count: result.modifiedCount });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to mark stale units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Update daily summary document.
+ * Upserts by date field (YYYY-MM-DD format).
+ * Fetches yesterday's summary for comparison metrics.
+ * Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} summaryData - Summary data
+ * @param {string} summaryData.date - Date in YYYY-MM-DD format
+ * @param {Array<string>} summaryData.newUnits - Unit codes added today
+ * @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
+ * @param {number} summaryData.staleUnitsCount - Count of stale units
+ * @param {number} summaryData.totalAvailable - Total available units today
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} MongoDB updateOne result
+ */
+async function updateDailySummary(db, summaryData, logger) {
+  const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
+  const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
+
+  // Calculate yesterday's date for comparison
+  const yesterday = getYesterday(date);
+
+  // Get yesterday's summary for comparison
+  const yesterdaySummary = await collection.findOne({ date: yesterday });
+  const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
+
+  const summary = {
+    date,
+    timestamp: new Date().toISOString(),
+    new_units: newUnits,
+    rented_units: rentedUnits,
+    stale_units: [], // Stale units list is not tracked per PRD
+    new_units_count: newUnits.length,
+    rented_units_count: rentedUnits.length,
+    stale_units_count: staleUnitsCount,
+    net_change: newUnits.length - rentedUnits.length,
+    total_available_today: totalAvailable,
+    total_available_yesterday: yesterdayTotal,
+    turnover_rate: yesterdayTotal > 0
+      ? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
+      : 0
+  };
+
+  try {
+    const result = await collection.updateOne(
+      { date },
+      { $set: summary },
+      { upsert: true }
+    );
+
+    logger.info('Daily summary updated', {
+      date,
+      newUnits: newUnits.length,
+      rentedUnits: rentedUnits.length,
+      totalAvailable
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to update daily summary', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+// ============================================================
+// Additional Helpers for runScrape Orchestration
+// ============================================================
+
+/**
+ * Get today's date in YYYY-MM-DD format (UTC).
+ * @returns {string} Today's date string
+ */
+function getTodayUTC() {
+  return new Date().toISOString().split('T')[0];
+}
+
+/**
+ * Get the set of unit codes that had price records yesterday.
+ * Used to calculate new and rented units by comparison.
+ * @param {Db} db - MongoDB database instance
+ * @param {string} today - Today's date in YYYY-MM-DD format
+ * @returns {Promise<Set<string>>} Set of unit codes from yesterday
+ */
+async function getYesterdayUnitCodes(db, today) {
+  const yesterday = getYesterday(today);
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+
+  const yesterdayRecords = await collection
+    .find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
+    .toArray();
+
+  return new Set(yesterdayRecords.map(r => r.unit_code));
+}
+
+// ============================================================
+// Scraper Run History
+// ============================================================
+
+/**
+ * Record scraper run to history collection.
+ * Called in the finally block of runScrape() to persist run metadata.
+ * This function must NOT throw errors - history recording should never break the scraper.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object|null>} Insert result, or null on failure
+ */
+async function recordScraperRun(db, runData, logger) {
+  try {
+    const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
+    const result = await collection.insertOne({
+      ...runData,
+      recordedAt: new Date()
+    });
+
+    return result;
+
+  } catch (error) {
+    // Log but don't throw - recording history should not break scraper
+    logger.error('Failed to record scraper run', { errorMessage: error.message });
+    return null;
+  }
+}
+
+// ============================================================
+// Main Orchestration Function
+// ============================================================
+
+/**
+ * Execute a complete scrape operation.
+ * Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} options - Scrape options
+ * @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
+ * @param {string} [options.jobId] - Optional job ID (generated if not provided)
+ * @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
+ * @param {string} [options.htmlContent] - Use provided HTML instead of fetching
+ * @returns {Promise<Object>} Scrape result with status and metrics
+ */
+async function runScrape(db, options = {}) {
+  const jobId = options.jobId || crypto.randomUUID();
+  const trigger = options.trigger || 'manual';
+  const dryRun = options.dryRun || false;
+  const htmlContent = options.htmlContent || null;
+  const logger = createLogger(jobId);
+  const startTime = Date.now();
+
+  let result = {
+    jobId,
+    trigger,
+    dryRun,
+    status: 'running',
+    startedAt: new Date().toISOString(),
+    completedAt: null,
+    duration: null,
+    unitsProcessed: 0,
+    pricesInserted: 0,
+    newUnitsCount: 0,
+    rentedUnitsCount: 0,
+    staleUnitsCount: 0,
+    errors: []
+  };
+
+  try {
+    logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
+
+    // Step 1: Fetch HTML (or use provided content for testing)
+    const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
+
+    // Step 2: Parse units
+    const rawUnits = parseUnits(html, logger);
+
+    if (rawUnits.length === 0) {
+      logger.warn('No units found in HTML - possible structure change');
+      result.errors.push('No units found in HTML');
+    }
+
+    // Step 3: Convert data types
+    const units = rawUnits.map(unit => convertDataTypes(unit));
+    result.unitsProcessed = units.length;
+
+    // Step 4: Database operations
+    const today = getTodayUTC();
+
+    // Get yesterday's unit codes for comparison
+    const yesterdayUnits = await getYesterdayUnitCodes(db, today);
+
+    // Determine new and rented units
+    const currentUnitCodes = new Set(units.map(u => u.unit_code));
+    const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
+    const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
+
+    result.newUnitsCount = newUnits.length;
+    result.rentedUnitsCount = rentedUnits.length;
+
+    // Database operations (skip if dryRun)
+    if (dryRun) {
+      logger.info('Dry run mode - skipping database writes', {
+        wouldUpsert: units.length,
+        wouldInsertPrices: units.filter(u => u.price !== null).length
+      });
+      result.pricesInserted = 0;
+      result.staleUnitsCount = 0;
+    } else {
+      // Upsert units
+      await upsertUnits(db, units, logger);
+
+      // Insert prices
+      const pricesResult = await insertPrices(db, units, today, logger);
+      result.pricesInserted = pricesResult.insertedCount;
+
+      // Mark stale units
+      const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
+      result.staleUnitsCount = staleResult.modifiedCount;
+
+      // Update daily summary
+      await updateDailySummary(db, {
+        date: today,
+        newUnits: newUnits.map(u => u.unit_code),
+        rentedUnits,
+        staleUnitsCount: result.staleUnitsCount,
+        totalAvailable: units.length
+      }, logger);
+    }
+
+    result.status = 'success';
+
+  } catch (error) {
+    logger.error('Scrape failed', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    result.status = 'failed';
+    result.errors.push(error.message);
+
+  } finally {
+    result.completedAt = new Date().toISOString();
+    result.duration = Date.now() - startTime;
+
+    // Record run to history (always runs, even on failure)
+    await recordScraperRun(db, result, logger);
+
+    logger.info('Scrape completed', {
+      status: result.status,
+      duration: result.duration,
+      unitsProcessed: result.unitsProcessed,
+      pricesInserted: result.pricesInserted
+    });
+  }
+
+  return result;
+}
+
+module.exports = {
+  runScrape,
+  fetchPage,
+  parseUnits,
+  convertDataTypes,
+  upsertUnits,
+  insertPrices,
+  markStaleUnits,
+  updateDailySummary,
+  recordScraperRun,
+  // Export helpers for testing
+  getTodayUTC,
+  getYesterdayUnitCodes,
+  getYesterday,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString,
+  isRetryableError,
+  sleep
+};

test-endpoints.js

@@ -0,0 +1,680 @@
+#!/usr/bin/env node
+
+/**
+ * Comprehensive API Endpoint Test Script
+ * Tests all apartment dashboard endpoints with enhanced redirect handling
+ * Usage: node test-endpoints.js
+ */
+
+const https = require('https');
+const http = require('http');
+
+const BASE_URL = 'apartments.maverickapplications.com';
+const API_PREFIX = '/api';
+
+// ANSI color codes for console output
+const colors = {
+  reset: '\x1b[0m',
+  bright: '\x1b[1m',
+  red: '\x1b[31m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  magenta: '\x1b[35m',
+  cyan: '\x1b[36m'
+};
+
+// Test results tracking
+let testResults = {
+  passed: 0,
+  failed: 0,
+  total: 0,
+  details: []
+};
+
+// Configuration for different protocols
+let currentConfig = {
+  protocol: 'http',
+  port: 80,
+  module: http
+};
+
+/**
+ * Make HTTP/HTTPS request with redirect handling
+ */
+function makeRequest(path, method = 'GET', maxRedirects = 5) {
+  return new Promise((resolve, reject) => {
+    function attemptRequest(currentPath, redirectCount = 0) {
+      const options = {
+        hostname: BASE_URL,
+        port: currentConfig.port,
+        path: currentPath,
+        method: method,
+        headers: {
+          'User-Agent': 'Apartment-API-Tester/1.0',
+          'Accept': 'application/json',
+          'Host': BASE_URL
+        },
+        timeout: 15000 // 15 second timeout
+      };
+
+      const req = currentConfig.module.request(options, (res) => {
+        let data = '';
+        
+        // Handle redirects
+        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          if (redirectCount >= maxRedirects) {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: null,
+              rawData: `Too many redirects (${redirectCount})`,
+              redirectLocation: res.headers.location
+            });
+            return;
+          }
+          
+          let redirectUrl = res.headers.location;
+          console.log(`${colors.yellow}    → Redirect ${res.statusCode} to: ${redirectUrl}${colors.reset}`);
+          
+          // Handle relative redirects
+          if (redirectUrl.startsWith('/')) {
+            attemptRequest(redirectUrl, redirectCount + 1);
+            return;
+          }
+          
+          // Handle absolute redirects (change protocol if needed)
+          if (redirectUrl.startsWith('https://') && currentConfig.protocol === 'http') {
+            console.log(`${colors.yellow}    → Switching to HTTPS due to redirect${colors.reset}`);
+            currentConfig = { protocol: 'https', port: 443, module: https };
+            const urlPath = redirectUrl.replace(`https://${BASE_URL}`, '');
+            attemptRequest(urlPath, redirectCount + 1);
+            return;
+          }
+          
+          if (redirectUrl.startsWith('http://') && currentConfig.protocol === 'https') {
+            console.log(`${colors.yellow}    → Switching to HTTP due to redirect${colors.reset}`);
+            currentConfig = { protocol: 'http', port: 80, module: http };
+            const urlPath = redirectUrl.replace(`http://${BASE_URL}`, '');
+            attemptRequest(urlPath, redirectCount + 1);
+            return;
+          }
+          
+          // For other absolute URLs, extract the path
+          try {
+            const url = new URL(redirectUrl);
+            if (url.hostname === BASE_URL) {
+              attemptRequest(url.pathname + url.search, redirectCount + 1);
+              return;
+            }
+          } catch (e) {
+            // If URL parsing fails, treat as relative
+            attemptRequest(redirectUrl, redirectCount + 1);
+            return;
+          }
+        }
+        
+        res.on('data', (chunk) => {
+          data += chunk;
+        });
+        
+        res.on('end', () => {
+          try {
+            const jsonData = data ? JSON.parse(data) : {};
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: jsonData,
+              rawData: data,
+              finalUrl: `${currentConfig.protocol}://${BASE_URL}${currentPath}`
+            });
+          } catch (e) {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              data: null,
+              rawData: data,
+              parseError: e.message,
+              finalUrl: `${currentConfig.protocol}://${BASE_URL}${currentPath}`
+            });
+          }
+        });
+      });
+
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error(`Request timeout after 15s for ${currentConfig.protocol}://${BASE_URL}${currentPath}`));
+      });
+
+      req.on('error', (error) => {
+        reject(new Error(`${error.message} (${currentConfig.protocol}://${BASE_URL}${currentPath})`));
+      });
+
+      req.end();
+    }
+    
+    attemptRequest(path);
+  });
+}
+
+/**
+ * Test connection and protocol detection
+ */
+async function detectBestProtocol() {
+  console.log(`${colors.magenta}🔍 Detecting best protocol and configuration...${colors.reset}\n`);
+  
+  const testConfigs = [
+    { protocol: 'http', port: 80, module: http, name: 'HTTP (port 80)' },
+    { protocol: 'https', port: 443, module: https, name: 'HTTPS (port 443)' },
+    { protocol: 'http', port: 8080, module: http, name: 'HTTP (port 8080)' },
+    { protocol: 'http', port: 3000, module: http, name: 'HTTP (port 3000)' }
+  ];
+  
+  for (const config of testConfigs) {
+    currentConfig = config;
+    console.log(`${colors.cyan}Testing ${config.name}...${colors.reset}`);
+    
+    try {
+      const response = await makeRequest('/health');
+      console.log(`${colors.green}✅ ${config.name} - Status: ${response.statusCode}${colors.reset}`);
+      
+      if (response.statusCode === 200) {
+        console.log(`${colors.green}🎯 Using ${config.name} for all tests${colors.reset}\n`);
+        return true;
+      } else if (response.statusCode >= 300 && response.statusCode < 400) {
+        console.log(`${colors.yellow}⚠️  ${config.name} redirects to: ${response.headers.location || 'unknown'}${colors.reset}`);
+        // Don't return, try other configs first
+      }
+    } catch (error) {
+      console.log(`${colors.red}❌ ${config.name} - ${error.message}${colors.reset}`);
+    }
+  }
+  
+  // If no 200 response found, use the first config that doesn't error
+  console.log(`${colors.yellow}⚠️  No perfect match found, using HTTP as fallback${colors.reset}\n`);
+  currentConfig = testConfigs[0];
+  return false;
+}
+
+/**
+ * Test a single endpoint with enhanced debugging
+ */
+async function testEndpoint(name, path, expectedFields = [], validStatusCodes = [200]) {
+  testResults.total++;
+  console.log(`${colors.cyan}Testing: ${colors.bright}${name}${colors.reset}`);
+  console.log(`${colors.blue}  Path: ${currentConfig.protocol}://${BASE_URL}${path}${colors.reset}`);
+  
+  try {
+    const response = await makeRequest(path);
+    const { statusCode, data, rawData, parseError, finalUrl, redirectLocation } = response;
+    
+    // Show final URL if different from initial
+    if (finalUrl && finalUrl !== `${currentConfig.protocol}://${BASE_URL}${path}`) {
+      console.log(`${colors.yellow}  Final URL: ${finalUrl}${colors.reset}`);
+    }
+    
+    // Show redirect info for debugging
+    if (statusCode >= 300 && statusCode < 400) {
+      console.log(`${colors.yellow}  Redirect Status: ${statusCode}${colors.reset}`);
+      if (redirectLocation) {
+        console.log(`${colors.yellow}  Redirect Location: ${redirectLocation}${colors.reset}`);
+      }
+    }
+    
+    // Check status code
+    if (!validStatusCodes.includes(statusCode)) {
+      // For 301/302 redirects, provide helpful info
+      if (statusCode === 301 || statusCode === 302) {
+        throw new Error(`Redirect ${statusCode} - Server redirecting to: ${redirectLocation || 'unknown'}. This might indicate wrong protocol or path format.`);
+      }
+      throw new Error(`Expected status ${validStatusCodes.join(' or ')}, got ${statusCode}`);
+    }
+    
+    // Check if response is valid JSON
+    if (parseError) {
+      console.log(`${colors.yellow}  Raw response: ${rawData.substring(0, 200)}...${colors.reset}`);
+      throw new Error(`JSON parse error: ${parseError}`);
+    }
+    
+    // Check for expected fields if data is an object
+    if (expectedFields.length > 0 && data && typeof data === 'object') {
+      const missingFields = [];
+      
+      if (Array.isArray(data)) {
+        // If it's an array, check the first item
+        if (data.length > 0) {
+          expectedFields.forEach(field => {
+            if (!(field in data[0])) {
+              missingFields.push(field);
+            }
+          });
+        }
+      } else {
+        // If it's an object, check directly
+        expectedFields.forEach(field => {
+          if (!(field in data)) {
+            missingFields.push(field);
+          }
+        });
+      }
+      
+      if (missingFields.length > 0) {
+        console.log(`${colors.yellow}  ⚠️  Missing expected fields: ${missingFields.join(', ')}${colors.reset}`);
+      }
+    }
+    
+    // Success
+    testResults.passed++;
+    console.log(`${colors.green}  ✅ PASS${colors.reset}`);
+    console.log(`${colors.green}     Status: ${statusCode}${colors.reset}`);
+    console.log(`${colors.green}     Protocol: ${currentConfig.protocol.toUpperCase()}${colors.reset}`);
+    
+    if (Array.isArray(data)) {
+      console.log(`${colors.green}     Records: ${data.length}${colors.reset}`);
+    } else if (data && typeof data === 'object') {
+      console.log(`${colors.green}     Keys: ${Object.keys(data).length}${colors.reset}`);
+    }
+    
+    testResults.details.push({
+      name,
+      path,
+      status: 'PASS',
+      statusCode,
+      protocol: currentConfig.protocol,
+      dataType: Array.isArray(data) ? 'array' : typeof data,
+      recordCount: Array.isArray(data) ? data.length : null
+    });
+    
+  } catch (error) {
+    testResults.failed++;
+    console.log(`${colors.red}  ❌ FAIL${colors.reset}`);
+    console.log(`${colors.red}     Error: ${error.message}${colors.reset}`);
+    console.log(`${colors.red}     Protocol: ${currentConfig.protocol.toUpperCase()}${colors.reset}`);
+    
+    testResults.details.push({
+      name,
+      path,
+      status: 'FAIL',
+      protocol: currentConfig.protocol,
+      error: error.message
+    });
+  }
+  
+  console.log(''); // Empty line for readability
+}
+
+/**
+ * Test with curl-equivalent settings
+ */
+async function testWithCurlEquivalent() {
+  console.log(`${colors.yellow}🔄 Testing with curl-equivalent settings...${colors.reset}\n`);
+  
+  // Test the exact same way curl would
+  const curlTests = [
+    { url: 'http://apartments.maverickapplications.com/health', description: 'Direct HTTP health check' },
+    { url: 'https://apartments.maverickapplications.com/health', description: 'Direct HTTPS health check' },
+    { url: 'http://apartments.maverickapplications.com/api/daily-summary', description: 'HTTP API endpoint' },
+    { url: 'https://apartments.maverickapplications.com/api/daily-summary', description: 'HTTPS API endpoint' }
+  ];
+  
+  for (const test of curlTests) {
+    console.log(`${colors.cyan}Testing: ${test.description}${colors.reset}`);
+    console.log(`${colors.blue}URL: ${test.url}${colors.reset}`);
+    
+    try {
+      const isHttps = test.url.startsWith('https');
+      const url = new URL(test.url);
+      
+      currentConfig = {
+        protocol: isHttps ? 'https' : 'http',
+        port: isHttps ? 443 : 80,
+        module: isHttps ? https : http
+      };
+      
+      const response = await makeRequest(url.pathname + url.search);
+      
+      console.log(`${colors.green}✅ Success - Status: ${response.statusCode}${colors.reset}`);
+      if (response.statusCode === 200 && response.data) {
+        console.log(`${colors.green}   Data type: ${Array.isArray(response.data) ? 'array' : typeof response.data}${colors.reset}`);
+        if (Array.isArray(response.data)) {
+          console.log(`${colors.green}   Records: ${response.data.length}${colors.reset}`);
+        } else if (typeof response.data === 'object') {
+          console.log(`${colors.green}   Keys: ${Object.keys(response.data).join(', ')}${colors.reset}`);
+        }
+      }
+      
+      // If we found a working endpoint, use this config for remaining tests
+      if (response.statusCode === 200) {
+        console.log(`${colors.green}🎯 Found working configuration: ${currentConfig.protocol.toUpperCase()}${colors.reset}\n`);
+        return true;
+      }
+      
+    } catch (error) {
+      console.log(`${colors.red}❌ Failed: ${error.message}${colors.reset}`);
+    }
+    console.log('');
+  }
+  
+  return false;
+}
+
+/**
+ * Test all endpoints
+ */
+async function runAllTests() {
+  console.log(`${colors.magenta}${colors.bright}🧪 APARTMENT API ENDPOINT TESTS${colors.reset}`);
+  console.log(`${colors.magenta}Base URL: ${BASE_URL}${colors.reset}`);
+  console.log(`${colors.magenta}Testing ${new Date().toISOString()}${colors.reset}\n`);
+  
+  // First, detect the best protocol
+  await detectBestProtocol();
+  
+  // Test a few different path formats to see which works
+  console.log(`${colors.magenta}🔍 Testing different path formats...${colors.reset}\n`);
+  
+  const pathVariations = [
+    '/health',
+    '/api/health', 
+    '/',
+    '/api/'
+  ];
+  
+  for (const testPath of pathVariations) {
+    try {
+      console.log(`${colors.cyan}Testing path: ${testPath}${colors.reset}`);
+      const response = await makeRequest(testPath);
+      console.log(`${colors.green}  Status: ${response.statusCode}${colors.reset}`);
+      if (response.statusCode === 200 && response.data) {
+        console.log(`${colors.green}  Response preview: ${JSON.stringify(response.data).substring(0, 100)}...${colors.reset}`);
+      }
+      if (response.rawData && response.statusCode !== 200) {
+        console.log(`${colors.yellow}  Raw response: ${response.rawData.substring(0, 200)}...${colors.reset}`);
+      }
+    } catch (error) {
+      console.log(`${colors.red}  Error: ${error.message}${colors.reset}`);
+    }
+    console.log('');
+  }
+  
+  console.log(`${colors.magenta}🚀 Starting full endpoint tests...${colors.reset}\n`);
+  
+  // Health check (try both with and without /api prefix)
+  await testEndpoint(
+    'Health Check',
+    '/health',
+    ['status', 'timestamp'],
+    [200, 404]
+  );
+  
+  // Try health check with /api prefix if first one failed
+  if (testResults.details[testResults.details.length - 1].status === 'FAIL') {
+    await testEndpoint(
+      'Health Check (with /api)',
+      '/api/health',
+      ['status', 'timestamp'],
+      [200, 404]
+    );
+  }
+  
+  // Original endpoints
+  await testEndpoint(
+    'Daily Summary',
+    `${API_PREFIX}/daily-summary`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Price History',
+    `${API_PREFIX}/price-history`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Available Units',
+    `${API_PREFIX}/available-units`,
+    ['unitCode'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Recent Activity',
+    `${API_PREFIX}/recent-activity`,
+    ['type'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Plan Statistics',
+    `${API_PREFIX}/plan-stats`,
+    ['name'],
+    [200, 404]
+  );
+  
+  // Enhanced endpoints (these might not exist yet)
+  await testEndpoint(
+    'Best Deals',
+    `${API_PREFIX}/best-deals`,
+    ['unitCode', 'planName', 'currentPrice'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Price Drops',
+    `${API_PREFIX}/price-drops`,
+    ['unitCode', 'currentPrice'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Stale Inventory',
+    `${API_PREFIX}/stale-inventory`,
+    ['unitCode', 'daysAvailable'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Market Insights',
+    `${API_PREFIX}/market-insights`,
+    ['avgDaysOnMarket'],
+    [200, 404, 500]
+  );
+  
+  await testEndpoint(
+    'Enhanced Available Units',
+    `${API_PREFIX}/available-units-enhanced`,
+    ['unitCode', 'amenities'],
+    [200, 404, 500]
+  );
+  
+  // Test with query parameters
+  await testEndpoint(
+    'Price History (7 days)',
+    `${API_PREFIX}/price-history?days=7`,
+    ['date'],
+    [200, 404]
+  );
+  
+  await testEndpoint(
+    'Best Deals (limit 3)',
+    `${API_PREFIX}/best-deals?limit=3`,
+    [],
+    [200, 404, 500]
+  );
+  
+  // Test individual unit endpoint
+  await testEndpoint(
+    'Individual Unit Details',
+    `${API_PREFIX}/unit/A101`,
+    [],
+    [200, 404, 500]
+  );
+  
+  // Test edge cases
+  await testEndpoint(
+    'Invalid Endpoint',
+    `${API_PREFIX}/nonexistent`,
+    [],
+    [404, 405] // Should return 404 or 405
+  );
+  
+  // Print final results
+  printTestSummary();
+}
+
+/**
+ * Print test summary
+ */
+function printTestSummary() {
+  console.log(`${colors.magenta}${colors.bright}📊 TEST SUMMARY${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  console.log(`${colors.green}✅ Passed: ${testResults.passed}${colors.reset}`);
+  console.log(`${colors.red}❌ Failed: ${testResults.failed}${colors.reset}`);
+  console.log(`${colors.blue}📋 Total:  ${testResults.total}${colors.reset}`);
+  console.log(`${colors.yellow}📈 Success Rate: ${((testResults.passed / testResults.total) * 100).toFixed(1)}%${colors.reset}`);
+  console.log('');
+  
+  // Detailed results
+  console.log(`${colors.magenta}${colors.bright}📋 DETAILED RESULTS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  
+  testResults.details.forEach((test, index) => {
+    const status = test.status === 'PASS' ? 
+      `${colors.green}✅ PASS${colors.reset}` : 
+      `${colors.red}❌ FAIL${colors.reset}`;
+    
+    console.log(`${index + 1}. ${test.name}`);
+    console.log(`   ${status} - ${test.path}`);
+    
+    if (test.status === 'PASS') {
+      if (test.recordCount !== null) {
+        console.log(`   📊 ${test.recordCount} records returned`);
+      }
+    } else {
+      console.log(`   💥 ${test.error}`);
+    }
+    console.log('');
+  });
+  
+  // Recommendations
+  console.log(`${colors.magenta}${colors.bright}💡 RECOMMENDATIONS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  
+  const failedTests = testResults.details.filter(t => t.status === 'FAIL');
+  const enhancedEndpoints = failedTests.filter(t => 
+    t.path.includes('best-deals') || 
+    t.path.includes('price-drops') || 
+    t.path.includes('stale-inventory') || 
+    t.path.includes('market-insights') || 
+    t.path.includes('available-units-enhanced')
+  );
+  
+  if (enhancedEndpoints.length > 0) {
+    console.log(`${colors.yellow}⚠️  Enhanced endpoints not implemented yet:${colors.reset}`);
+    enhancedEndpoints.forEach(test => {
+      console.log(`   - ${test.path}`);
+    });
+    console.log(`${colors.blue}   💡 Add the missing routes from the enhanced server script${colors.reset}`);
+    console.log('');
+  }
+  
+  if (testResults.failed > enhancedEndpoints.length) {
+    console.log(`${colors.red}🚨 Core endpoints failing - check server configuration${colors.reset}`);
+    console.log('');
+  }
+  
+  if (testResults.passed === testResults.total) {
+    console.log(`${colors.green}🎉 All tests passed! API is fully functional.${colors.reset}`);
+  }
+}
+
+/**
+ * Debug curl command generator
+ */
+function generateCurlCommands() {
+  console.log(`${colors.magenta}${colors.bright}🔧 EQUIVALENT CURL COMMANDS${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════${colors.reset}`);
+  console.log(`${colors.yellow}Try these curl commands to debug:${colors.reset}\n`);
+  
+  const endpoints = [
+    '/health',
+    '/api/health', 
+    '/api/daily-summary',
+    '/api/price-history',
+    '/api/available-units'
+  ];
+  
+  endpoints.forEach(endpoint => {
+    console.log(`${colors.cyan}# Test ${endpoint}${colors.reset}`);
+    console.log(`curl -v -H "Accept: application/json" http://${BASE_URL}${endpoint}`);
+    console.log(`curl -v -H "Accept: application/json" https://${BASE_URL}${endpoint}`);
+    console.log('');
+  });
+  
+  console.log(`${colors.yellow}Look for:${colors.reset}`);
+  console.log(`${colors.blue}- HTTP status codes (200 = success, 301/302 = redirect, 404 = not found)${colors.reset}`);
+  console.log(`${colors.blue}- Location headers in redirects${colors.reset}`);
+  console.log(`${colors.blue}- Response content-type${colors.reset}`);
+  console.log(`${colors.blue}- Server response headers${colors.reset}\n`);
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  try {
+    await runAllTests();
+    
+    // If all tests failed with redirects, try curl-equivalent testing
+    const allFailed = testResults.passed === 0 && testResults.failed > 0;
+    const hasRedirectErrors = testResults.details.some(test => 
+      test.error && test.error.includes('Redirect')
+    );
+    
+    if (allFailed && hasRedirectErrors) {
+      console.log(`${colors.yellow}🔍 All tests failed with redirects. Trying curl-equivalent approach...${colors.reset}\n`);
+      await testWithCurlEquivalent();
+    }
+    
+    // Generate curl commands for manual testing
+    generateCurlCommands();
+    
+  } catch (error) {
+    console.error(`${colors.red}💥 Fatal error: ${error.message}${colors.reset}`);
+    process.exit(1);
+  }
+  
+  // Exit with error code if tests failed
+  process.exit(testResults.failed > 0 ? 1 : 0);
+}
+
+// Handle graceful shutdown
+process.on('SIGINT', () => {
+  console.log(`\n${colors.yellow}🛑 Test interrupted by user${colors.reset}`);
+  process.exit(1);
+});
+
+// Add CLI argument parsing
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`
+${colors.bright}Apartment API Endpoint Tester${colors.reset}
+
+${colors.cyan}Usage:${colors.reset}
+  node test-endpoints.js [options]
+
+${colors.cyan}Options:${colors.reset}
+  --help, -h     Show this help message
+  --https        Force HTTPS testing
+  --verbose      Show detailed response data
+
+${colors.cyan}Examples:${colors.reset}
+  node test-endpoints.js
+  node test-endpoints.js --https
+  node test-endpoints.js --verbose
+`);
+  process.exit(0);
+}
+
+// Run the main function
+main();