Update parseUnits tests to match real fixture data

The fixture file was updated with real website data (10 units) but
the tests still referenced old synthetic data (5 units with CCT-*
codes). Updated tests to use real unit codes from the fixture (W2707,
E3205, W2603) and moved edge case tests (image extraction, missing
attributes, unavailable units) to use inline HTML for isolation.

.github/workflows/deploy.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
   workflow_dispatch:
 
 env:
@@ -13,22 +12,17 @@ env:
 
 jobs:
   # ============================================================
-  # Test Job - Runs first, blocks everything if tests fail
+  # CI Job - Lint + Test in a single job (one checkout, one npm ci)
   # ============================================================
-  test:
+  ci:
-    name: Run Tests
+    name: Lint & Test
     runs-on: ubuntu-latest
 
-    services:
+    outputs:
-      mongodb:
+      lint_status: ${{ steps.lint.outcome }}
-        image: mongo:7
+      lint_output: ${{ steps.lint.outputs.lint_output }}
-        ports:
+      test_status: ${{ steps.test.outcome }}
-          - 27018:27017
+      test_output: ${{ steps.test.outputs.test_output }}
-        options: >-
-          --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
 
     steps:
       - name: Checkout code
@@ -43,13 +37,137 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Run ESLint
+        id: lint
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm run lint 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+
       - name: Run tests
-        run: npm test -- --runInBand
+        id: test
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm test -- --runInBand 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "test_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
         env:
-          MONGO_URI: mongodb://localhost:27018
           JWT_SECRET: test-jwt-secret-for-ci
           NODE_ENV: test
 
+  # ============================================================
+  # Notify Job - Send results to n8n webhook
+  # ============================================================
+  notify:
+    name: Send Webhook Notification
+    runs-on: ubuntu-latest
+    needs: [ci]
+    if: always()
+
+    steps:
+      - name: Send results to n8n webhook
+        env:
+          LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
+          TEST_STATUS: ${{ needs.ci.outputs.test_status }}
+          RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
+          RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
+          GH_REPO: ${{ github.repository }}
+          GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+          GH_SHA: ${{ github.sha }}
+          GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
+          GH_ACTOR: ${{ github.actor }}
+          GH_EVENT: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
+        run: |
+          # Determine overall status
+          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
+            OVERALL_STATUS="success"
+          else
+            OVERALL_STATUS="failure"
+          fi
+
+          # Outputs are already truncated at the source (ci job)
+          LINT_OUTPUT="$RAW_LINT_OUTPUT"
+          TEST_OUTPUT="$RAW_TEST_OUTPUT"
+
+          # Build JSON payload
+          PAYLOAD=$(jq -n \
+            --arg repo "$GH_REPO" \
+            --arg branch "$GH_BRANCH" \
+            --arg commit "$GH_SHA" \
+            --arg commit_short "${GH_SHA:0:7}" \
+            --arg commit_message "$GH_COMMIT_MSG" \
+            --arg author "$GH_ACTOR" \
+            --arg event "$GH_EVENT" \
+            --arg pr_number "$GH_PR_NUMBER" \
+            --arg run_id "$GH_RUN_ID" \
+            --arg run_url "$GH_RUN_URL" \
+            --arg overall_status "$OVERALL_STATUS" \
+            --arg lint_status "$LINT_STATUS" \
+            --arg lint_output "$LINT_OUTPUT" \
+            --arg test_status "$TEST_STATUS" \
+            --arg test_output "$TEST_OUTPUT" \
+            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            '{
+              repository: $repo,
+              branch: $branch,
+              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
+              commit: {
+                sha: $commit,
+                short_sha: $commit_short,
+                message: $commit_message,
+                author: $author
+              },
+              event: $event,
+              run: {
+                id: $run_id,
+                url: $run_url
+              },
+              status: $overall_status,
+              results: {
+                lint: {
+                  status: $lint_status,
+                  output: (if $lint_status != "success" then $lint_output else null end)
+                },
+                test: {
+                  status: $test_status,
+                  output: (if $test_status != "success" then $test_output else null end)
+                }
+              },
+              timestamp: $timestamp
+            }')
+
+          # Send webhook
+          curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "$WEBHOOK_URL" \
+            --fail --silent --show-error
+
+      - name: Fail if lint or tests failed
+        if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "   Lint: ${{ needs.ci.outputs.lint_status }}"
+          echo "   Test: ${{ needs.ci.outputs.test_status }}"
+          exit 1
+
   # ============================================================
   # Dependency Scan Job - Runs in parallel with tests
   # ============================================================
@@ -80,7 +198,7 @@ jobs:
   build:
     name: Build & Push Image
     runs-on: ubuntu-latest
-    needs: [test, scan-deps]
+    needs: [ci, scan-deps, notify]
     if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
 
     outputs:

__tests__/scraper/dataConversion.test.js

@@ -0,0 +1,496 @@
+/**
+ * Tests for convertDataTypes() and its internal helper functions
+ *
+ * Covers:
+ * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
+ * - parseInteger: valid integers, null/empty/invalid
+ * - parsePositiveInteger: treats 0 as null (for area)
+ * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
+ * - parseFloat: "1.5" for half baths, null/empty/invalid
+ * - parseBoolean: "true"/"false"/"1"/"0", null/empty
+ * - trimString: trim whitespace, null for empty, null/undefined
+ * - convertDataTypes: full integration applying correct parser to each field
+ */
+
+const {
+  convertDataTypes,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString
+} = require('../../services/scraperService');
+
+// ---------------------------------------------------------------
+// 1. parsePrice
+// ---------------------------------------------------------------
+describe('parsePrice', () => {
+  it('should parse "$1,234" format to 1234', () => {
+    expect(parsePrice('$1,234')).toBe(1234);
+  });
+
+  it('should parse plain numeric string', () => {
+    expect(parsePrice('1450')).toBe(1450);
+  });
+
+  it('should parse "$2,500.00" stripping non-numeric chars', () => {
+    expect(parsePrice('$2,500.00')).toBe(2500);
+  });
+
+  it('should return null for "Call for pricing"', () => {
+    expect(parsePrice('Call for pricing')).toBeNull();
+  });
+
+  it('should return null for "Contact us"', () => {
+    expect(parsePrice('Contact us')).toBeNull();
+  });
+
+  it('should return null for "Please inquire"', () => {
+    expect(parsePrice('Please inquire')).toBeNull();
+  });
+
+  it('should return null for negative values', () => {
+    // After stripping non-numeric chars, "-1500" becomes "1500"
+    // but if someone passes a numeric -1500 directly, check that too
+    expect(parsePrice('-')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(parsePrice(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePrice('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parsePrice(undefined)).toBeNull();
+  });
+
+  it('should return null for completely non-numeric string', () => {
+    expect(parsePrice('abc')).toBeNull();
+  });
+
+  it('should handle price of 0', () => {
+    expect(parsePrice('0')).toBe(0);
+  });
+
+  it('should handle large prices', () => {
+    expect(parsePrice('$15,500')).toBe(15500);
+  });
+});
+
+// ---------------------------------------------------------------
+// 2. parseInteger
+// ---------------------------------------------------------------
+describe('parseInteger', () => {
+  it('should parse valid integer string', () => {
+    expect(parseInteger('42')).toBe(42);
+  });
+
+  it('should parse "0" to 0', () => {
+    expect(parseInteger('0')).toBe(0);
+  });
+
+  it('should parse negative integer string', () => {
+    expect(parseInteger('-5')).toBe(-5);
+  });
+
+  it('should return null for null', () => {
+    expect(parseInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseInteger('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseInteger(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseInteger('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseInteger('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseInteger('Infinity')).toBeNull();
+  });
+
+  it('should truncate float strings to integer', () => {
+    expect(parseInteger('3.7')).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------
+// 3. parsePositiveInteger
+// ---------------------------------------------------------------
+describe('parsePositiveInteger', () => {
+  it('should parse positive integer normally', () => {
+    expect(parsePositiveInteger('750')).toBe(750);
+  });
+
+  it('should treat 0 as null (for area field)', () => {
+    expect(parsePositiveInteger('0')).toBeNull();
+  });
+
+  it('should return null for null input', () => {
+    expect(parsePositiveInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePositiveInteger('')).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parsePositiveInteger('abc')).toBeNull();
+  });
+
+  it('should handle negative values (parsed as negative int, not positive)', () => {
+    expect(parsePositiveInteger('-5')).toBe(-5);
+  });
+});
+
+// ---------------------------------------------------------------
+// 4. parseIntegerOrString
+// ---------------------------------------------------------------
+describe('parseIntegerOrString', () => {
+  it('should return integer when value is cleanly parseable', () => {
+    expect(parseIntegerOrString('5001')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric value', () => {
+    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
+  });
+
+  it('should return trimmed string when value has non-numeric suffix', () => {
+    expect(parseIntegerOrString('123abc')).toBe('123abc');
+  });
+
+  it('should return null for null', () => {
+    expect(parseIntegerOrString(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseIntegerOrString('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseIntegerOrString(undefined)).toBeNull();
+  });
+
+  it('should trim whitespace before checking', () => {
+    expect(parseIntegerOrString('  5001  ')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric with whitespace', () => {
+    expect(parseIntegerOrString('  abc  ')).toBe('abc');
+  });
+});
+
+// ---------------------------------------------------------------
+// 5. parseFloatValue
+// ---------------------------------------------------------------
+describe('parseFloatValue', () => {
+  it('should parse "1.5" for half baths', () => {
+    expect(parseFloatValue('1.5')).toBe(1.5);
+  });
+
+  it('should parse "2.0" to 2', () => {
+    expect(parseFloatValue('2.0')).toBe(2.0);
+  });
+
+  it('should parse integer string as float', () => {
+    expect(parseFloatValue('3')).toBe(3);
+  });
+
+  it('should return null for null', () => {
+    expect(parseFloatValue(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseFloatValue('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseFloatValue(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseFloatValue('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseFloatValue('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseFloatValue('Infinity')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 6. parseBoolean
+// ---------------------------------------------------------------
+describe('parseBoolean', () => {
+  it('should parse "true" to true', () => {
+    expect(parseBoolean('true')).toBe(true);
+  });
+
+  it('should parse "false" to false', () => {
+    expect(parseBoolean('false')).toBe(false);
+  });
+
+  it('should parse "1" to true', () => {
+    expect(parseBoolean('1')).toBe(true);
+  });
+
+  it('should parse "0" to false', () => {
+    expect(parseBoolean('0')).toBe(false);
+  });
+
+  it('should parse "TRUE" (case insensitive) to true', () => {
+    expect(parseBoolean('TRUE')).toBe(true);
+  });
+
+  it('should parse "False" (case insensitive) to false', () => {
+    expect(parseBoolean('False')).toBe(false);
+  });
+
+  it('should pass through boolean true', () => {
+    expect(parseBoolean(true)).toBe(true);
+  });
+
+  it('should pass through boolean false', () => {
+    expect(parseBoolean(false)).toBe(false);
+  });
+
+  it('should return null for null', () => {
+    expect(parseBoolean(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseBoolean('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseBoolean(undefined)).toBeNull();
+  });
+
+  it('should return null for unrecognized string', () => {
+    expect(parseBoolean('yes')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 7. trimString
+// ---------------------------------------------------------------
+describe('trimString', () => {
+  it('should trim leading and trailing whitespace', () => {
+    expect(trimString('  hello  ')).toBe('hello');
+  });
+
+  it('should return the string as-is if no whitespace', () => {
+    expect(trimString('hello')).toBe('hello');
+  });
+
+  it('should return null for empty string', () => {
+    expect(trimString('')).toBeNull();
+  });
+
+  it('should return null for whitespace-only string', () => {
+    expect(trimString('   ')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(trimString(null)).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(trimString(undefined)).toBeNull();
+  });
+
+  it('should convert non-string to string then trim', () => {
+    expect(trimString(123)).toBe('123');
+  });
+});
+
+// ---------------------------------------------------------------
+// 8. convertDataTypes - Full integration test
+// ---------------------------------------------------------------
+describe('convertDataTypes', () => {
+  it('should apply correct parser to each field', () => {
+    const rawUnit = {
+      id: '1001',
+      unit_code: 'CCT-101',
+      unit_id: '5001',
+      floor: '1',
+      area: '750',
+      bed_count: '1',
+      bath_count: '1.5',
+      price: '1450',
+      available: 'true',
+      unavailable: 'false',
+      soonest: '2026-03-01',
+      date_available: '20260301',
+      plan_id: '201',
+      plan_name: 'Alpine',
+      obj_type: 'unit',
+      community: 'Country Club Towers',
+      asset: '100',
+      href: '/units/CCT-101',
+      inventory_href: '/inventory/CCT-101',
+      image_url: 'https://example.com/images/unit-101.jpg',
+      specials_content: 'First month free!'
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    // Integer fields
+    expect(converted.id).toBe(1001);
+    expect(converted.floor).toBe(1);
+    expect(converted.bed_count).toBe(1);
+    expect(converted.plan_id).toBe(201);
+    expect(converted.asset).toBe(100);
+    expect(converted.date_available).toBe(20260301);
+
+    // parseIntegerOrString
+    expect(converted.unit_id).toBe(5001);
+
+    // parsePositiveInteger (area)
+    expect(converted.area).toBe(750);
+
+    // Float field (bath_count)
+    expect(converted.bath_count).toBe(1.5);
+
+    // Price field
+    expect(converted.price).toBe(1450);
+
+    // Boolean fields
+    expect(converted.available).toBe(true);
+    expect(converted.unavailable).toBe(false);
+
+    // String fields (trimmed)
+    expect(converted.unit_code).toBe('CCT-101');
+    expect(converted.plan_name).toBe('Alpine');
+    expect(converted.soonest).toBe('2026-03-01');
+    expect(converted.obj_type).toBe('unit');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.href).toBe('/units/CCT-101');
+    expect(converted.inventory_href).toBe('/inventory/CCT-101');
+    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
+    expect(converted.specials_content).toBe('First month free!');
+  });
+
+  it('should preserve string fields without converting them', () => {
+    const rawUnit = {
+      unit_code: 'CCT-205',
+      plan_name: '  Birch  ',
+      soonest: 'Now',
+      obj_type: 'unit',
+      community: '  Country Club Towers  ',
+      href: '/units/CCT-205',
+      inventory_href: '/inventory/CCT-205',
+      image_url: 'https://example.com/img.jpg',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.unit_code).toBe('CCT-205');
+    expect(converted.plan_name).toBe('Birch');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle a unit with all null/undefined/empty values', () => {
+    const rawUnit = {
+      id: null,
+      unit_code: undefined,
+      unit_id: '',
+      floor: null,
+      area: '',
+      bed_count: undefined,
+      bath_count: null,
+      price: '',
+      available: null,
+      unavailable: undefined,
+      soonest: '',
+      date_available: null,
+      plan_id: undefined,
+      plan_name: '',
+      obj_type: null,
+      community: undefined,
+      asset: '',
+      href: null,
+      inventory_href: undefined,
+      image_url: '',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.id).toBeNull();
+    expect(converted.unit_code).toBeNull();
+    expect(converted.unit_id).toBeNull();
+    expect(converted.floor).toBeNull();
+    expect(converted.area).toBeNull();
+    expect(converted.bed_count).toBeNull();
+    expect(converted.bath_count).toBeNull();
+    expect(converted.price).toBeNull();
+    expect(converted.available).toBeNull();
+    expect(converted.unavailable).toBeNull();
+    expect(converted.soonest).toBeNull();
+    expect(converted.date_available).toBeNull();
+    expect(converted.plan_id).toBeNull();
+    expect(converted.plan_name).toBeNull();
+    expect(converted.obj_type).toBeNull();
+    expect(converted.community).toBeNull();
+    expect(converted.asset).toBeNull();
+    expect(converted.href).toBeNull();
+    expect(converted.inventory_href).toBeNull();
+    expect(converted.image_url).toBeNull();
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle area=0 as null (parsePositiveInteger)', () => {
+    const rawUnit = { area: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.area).toBeNull();
+  });
+
+  it('should handle bed_count=0 as valid (studio)', () => {
+    const rawUnit = { bed_count: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bed_count).toBe(0);
+  });
+
+  it('should handle "Call for pricing" in price field', () => {
+    const rawUnit = { price: 'Call for pricing' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBeNull();
+  });
+
+  it('should handle "$1,234" price format', () => {
+    const rawUnit = { price: '$1,234' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBe(1234);
+  });
+
+  it('should handle unit_id as string identifier', () => {
+    const rawUnit = { unit_id: 'UNIT-ABC' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.unit_id).toBe('UNIT-ABC');
+  });
+
+  it('should handle bath_count with half baths', () => {
+    const rawUnit = { bath_count: '1.5' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bath_count).toBe(1.5);
+  });
+});

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

__tests__/scraper/fixtures/sample-listing-call.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Call for Pricing</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9001"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="750"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="P-101"
+    data-spaces-unit-id="9001"
+    data-spaces-unit-floor="1"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="500"
+    data-spaces-sort-plan-name="Penthouse A"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9001"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
+    aria-label="Unit P-101">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-101</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9002"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="900"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="P-102"
+    data-spaces-unit-id="9002"
+    data-spaces-unit-floor="2"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="501"
+    data-spaces-sort-plan-name="Penthouse B"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9002"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
+    aria-label="Unit P-102">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-102</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing-empty.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - No Units</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <!-- Empty - no units available -->
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Apartments</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154842"
+    data-spaces-soonest="2025-10-11"
+    data-spaces-sort-date="1760140800"
+    data-spaces-sort-price="5230"
+    data-spaces-sort-area="1210"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W2707"
+    data-spaces-unit-id="154842"
+    data-spaces-unit-floor="2755"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="10270"
+    data-spaces-sort-plan-name="Pyramid Peak - Terrace"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154842"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
+    aria-label="Unit W2707">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">5230</span>
+      <span class="spaces-unit__unit">W2707</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154451"
+    data-spaces-soonest="2026-03-08"
+    data-spaces-sort-date="1772928000"
+    data-spaces-sort-price="2767"
+    data-spaces-sort-area="806"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="E0901"
+    data-spaces-unit-id="154451"
+    data-spaces-unit-floor="2738"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8016"
+    data-spaces-sort-plan-name="Sunshine Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154451"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
+    aria-label="Unit E0901">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2767</span>
+      <span class="spaces-unit__unit">E0901</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154814"
+    data-spaces-soonest="2025-12-18"
+    data-spaces-sort-date="1766016000"
+    data-spaces-sort-price="2255"
+    data-spaces-sort-area="623"
+    data-spaces-sort-bed="0"
+    data-spaces-unit="W2603"
+    data-spaces-unit-id="154814"
+    data-spaces-unit-floor="2754"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8018"
+    data-spaces-sort-plan-name="Little Bear Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154814"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
+    aria-label="Unit W2603">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2255</span>
+      <span class="spaces-unit__unit">W2603</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154428"
+    data-spaces-soonest="2026-02-22"
+    data-spaces-sort-date="1771718400"
+    data-spaces-sort-price="4250"
+    data-spaces-sort-area="1152"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W0802"
+    data-spaces-unit-id="154428"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8017"
+    data-spaces-sort-plan-name="Mt. Princeton"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154428"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
+    aria-label="Unit W0802">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4250</span>
+      <span class="spaces-unit__unit">W0802</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154778"
+    data-spaces-soonest="2025-12-31"
+    data-spaces-sort-date="1767139200"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="764"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2405"
+    data-spaces-unit-id="154778"
+    data-spaces-unit-floor="2752"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8020"
+    data-spaces-sort-plan-name="Grays Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154778"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
+    aria-label="Unit W2405">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W2405</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154442"
+    data-spaces-soonest="2025-02-01"
+    data-spaces-sort-date="1738368000"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="802"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W0809"
+    data-spaces-unit-id="154442"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8022"
+    data-spaces-sort-plan-name="Pikes Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154442"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
+    aria-label="Unit W0809">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W0809</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154764"
+    data-spaces-soonest="2026-02-17"
+    data-spaces-sort-date="1771286400"
+    data-spaces-sort-price="2435"
+    data-spaces-sort-area="715"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2308"
+    data-spaces-unit-id="154764"
+    data-spaces-unit-floor="2751"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8023"
+    data-spaces-sort-plan-name="Longs Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154764"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
+    aria-label="Unit W2308">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2435</span>
+      <span class="spaces-unit__unit">W2308</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154612"
+    data-spaces-soonest="2026-03-24"
+    data-spaces-sort-date="1774310400"
+    data-spaces-sort-price="2683"
+    data-spaces-sort-area="797"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W1610"
+    data-spaces-unit-id="154612"
+    data-spaces-unit-floor="2744"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8024"
+    data-spaces-sort-plan-name="Torreys Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154612"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
+    aria-label="Unit W1610">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2683</span>
+      <span class="spaces-unit__unit">W1610</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154495"
+    data-spaces-soonest="2025-11-07"
+    data-spaces-sort-date="1762473600"
+    data-spaces-sort-price="4470"
+    data-spaces-sort-area="1230"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E1011"
+    data-spaces-unit-id="154495"
+    data-spaces-unit-floor="2739"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8025"
+    data-spaces-sort-plan-name="Maroon Peak"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154495"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
+    aria-label="Unit E1011">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4470</span>
+      <span class="spaces-unit__unit">E1011</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="154919"
+    data-spaces-soonest="2025-10-27"
+    data-spaces-sort-date="1761523200"
+    data-spaces-sort-price="8581"
+    data-spaces-sort-area="1532"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E3205"
+    data-spaces-unit-id="154919"
+    data-spaces-unit-floor="2760"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8037"
+    data-spaces-sort-plan-name="Mt. Antero"
+    data-spaces-sort-bath="2.5"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154919"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
+    aria-label="Unit E3205">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">8581</span>
+      <span class="spaces-unit__unit">E3205</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/insertPrices.test.js

@@ -0,0 +1,582 @@
+/**
+ * Tests for insertPrices() bulk operation
+ *
+ * Covers:
+ * - Only units with non-null prices are inserted
+ * - bulkWrite uses updateOne with upsert for idempotency
+ * - Filter uses unit_code + date_checked composite key
+ * - Price record includes unit_code, date_checked, price, last_updated, data_source
+ * - Empty price records logs warning and returns early
+ * - Error handling when bulkWrite fails
+ * - Logging of inserted and updated counts
+ * - Re-running on same day updates existing records (idempotent)
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require insertPrices after implementation
+let insertPrices;
+
+let mongoServer;
+let client;
+let db;
+
+const PRICES_COLLECTION = 'unit_prices_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ insertPrices } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the prices collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('insertPrices', () => {
+  const TODAY = '2026-02-05';
+
+  // ---------------------------------------------------------------
+  // 1. Only units with non-null prices are inserted
+  // ---------------------------------------------------------------
+  describe('filtering units with null prices', () => {
+    it('should only insert records for units with non-null prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200 },
+        { unit_code: 'A102', price: null },
+        { unit_code: 'A103', price: 1500 },
+        { unit_code: 'A104', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'A103']);
+    });
+
+    it('should skip units with price of null and not create any record for them', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'B101', price: null },
+        { unit_code: 'B102', price: null }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(0);
+    });
+
+    it('should handle a mix of priced and null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'C101', price: 950 },
+        { unit_code: 'C102', price: null },
+        { unit_code: 'C103', price: 1100 },
+        { unit_code: 'C104', price: null },
+        { unit_code: 'C105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. bulkWrite uses updateOne with upsert for idempotency
+  // ---------------------------------------------------------------
+  describe('bulkWrite with upsert operations', () => {
+    it('should create updateOne operations for each priced unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'D101', price: 1200 },
+        { unit_code: 'D102', price: 1500 }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.insertedCount).toBeGreaterThan(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+    });
+
+    it('should use upsert so re-running does not create duplicates', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'D201', price: 1200 }];
+
+      // First run
+      await insertPrices(db, units, TODAY, logger);
+
+      // Second run on same day
+      await insertPrices(db, units, TODAY, logger);
+
+      // Should still have only 1 document (not 2)
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Filter uses unit_code + date_checked composite key
+  // ---------------------------------------------------------------
+  describe('composite key: unit_code + date_checked', () => {
+    it('should use unit_code and date_checked as the filter for upsert', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E101', price: 1200 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'E101',
+        date_checked: TODAY
+      });
+
+      expect(doc).not.toBeNull();
+      expect(doc.unit_code).toBe('E101');
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should create separate records for the same unit on different days', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E201', price: 1200 }];
+
+      await insertPrices(db, units, '2026-02-04', logger);
+      await insertPrices(db, units, '2026-02-05', logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ unit_code: 'E201' })
+        .sort({ date_checked: 1 })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+      expect(docs[0].date_checked).toBe('2026-02-04');
+      expect(docs[1].date_checked).toBe('2026-02-05');
+    });
+
+    it('should create separate records for different units on the same day', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'E301', price: 1000 },
+        { unit_code: 'E302', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ date_checked: TODAY })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Price record includes required fields
+  // ---------------------------------------------------------------
+  describe('price record fields', () => {
+    it('should include unit_code in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.unit_code).toBe('F101');
+    });
+
+    it('should include date_checked in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F201', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should include price as a number in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F301', price: 1550 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.price).toBe(1550);
+      expect(typeof doc.price).toBe('number');
+    });
+
+    it('should include last_updated as an ISO timestamp string', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'F401', price: 1600 }];
+      await insertPrices(db, units, TODAY, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.last_updated).toBeDefined();
+      expect(typeof doc.last_updated).toBe('string');
+      expect(doc.last_updated >= beforeTime).toBe(true);
+      expect(doc.last_updated <= afterTime).toBe(true);
+    });
+
+    it('should include data_source set to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F501', price: 1700 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should include all five required fields in every price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F601', price: 1800 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc).toHaveProperty('unit_code');
+      expect(doc).toHaveProperty('date_checked');
+      expect(doc).toHaveProperty('price');
+      expect(doc).toHaveProperty('last_updated');
+      expect(doc).toHaveProperty('data_source');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty price records logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty price records', () => {
+    it('should log a warning when no units have prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'G101', price: null },
+        { unit_code: 'G102', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should return insertedCount of 0 for all null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'G201', price: null }];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+    });
+
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should not perform any database writes for empty/null-priced arrays', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H101', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H201', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await insertPrices(mockDb, units, TODAY, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to insert prices',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H301', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Logging of inserted and updated counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log inserted and updated counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I101', price: 1000 },
+        { unit_code: 'I102', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: expect.any(Number),
+          updated: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I201', price: 1000 },
+        { unit_code: 'I202', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // For new inserts: inserted=2, updated=0
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 2,
+          updated: 0
+        })
+      );
+    });
+
+    it('should log correct counts when updating existing records', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'I301', price: 1000 }];
+
+      // First insert
+      await insertPrices(db, units, TODAY, logger);
+
+      // Reset mock
+      logger.info.mockClear();
+
+      // Second run updates existing
+      const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
+      await insertPrices(db, updatedUnits, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 0,
+          updated: 1
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Re-running on same day updates existing records (idempotent)
+  // ---------------------------------------------------------------
+  describe('idempotent re-runs on same day', () => {
+    it('should update the price when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run with original price
+      await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
+
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc1.price).toBe(1200);
+
+      // Re-run with updated price (same day)
+      await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
+
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc2.price).toBe(1250);
+
+      // Should still have only 1 record
+      const count = await db.collection(PRICES_COLLECTION).countDocuments({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(count).toBe(1);
+    });
+
+    it('should update last_updated when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run
+      await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const firstUpdated = doc1.last_updated;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Re-run on same day
+      await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const secondUpdated = doc2.last_updated;
+
+      expect(secondUpdated > firstUpdated).toBe(true);
+    });
+
+    it('should return combined insert/update count reflecting the operation', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'J301', price: 1000 },
+        { unit_code: 'J302', price: 1100 }
+      ];
+
+      // First run - inserts
+      const result1 = await insertPrices(db, units, TODAY, logger);
+      expect(result1.insertedCount).toBe(2);
+
+      // Re-run - updates
+      const result2 = await insertPrices(db, units, TODAY, logger);
+      expect(result2.insertedCount).toBe(2); // combined upserted + modified
+
+      // Still only 2 total records
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(2);
+    });
+
+    it('should handle a mix of new and existing records on re-run', async () => {
+      const logger = createMockLogger();
+
+      // First run with unit A
+      await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
+
+      // Re-run with unit A (existing) and unit B (new)
+      const mixedUnits = [
+        { unit_code: 'J401', price: 1050 },
+        { unit_code: 'J402', price: 1200 }
+      ];
+      await insertPrices(db, mixedUnits, TODAY, logger);
+
+      // Should have 2 total records
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      // Verify updated price
+      const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
+      expect(unitA.price).toBe(1050);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Uses ordered: false for performance
+  // ---------------------------------------------------------------
+  describe('ordered: false for performance', () => {
+    it('should successfully process all priced units in a batch', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'K101', price: 900 },
+        { unit_code: 'K102', price: 1000 },
+        { unit_code: 'K103', price: 1100 },
+        { unit_code: 'K104', price: 1200 },
+        { unit_code: 'K105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Uses correct collection
+  // ---------------------------------------------------------------
+  describe('collection usage', () => {
+    it('should write to the configured PRICES collection', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'L101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // Verify data is in the correct collection
+      const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
+      expect(doc).not.toBeNull();
+    });
+  });
+});

__tests__/scraper/markStaleUnits.test.js

@@ -0,0 +1,449 @@
+/**
+ * Tests for markStaleUnits() availability tracking
+ *
+ * Covers:
+ * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
+ * - $set updates available to false and sets marked_stale_date
+ * - Units in currentCodes are NOT marked stale
+ * - Already unavailable units are not modified
+ * - Error handling when updateMany fails
+ * - Logging of modified count
+ * - Empty currentUnitCodes handles gracefully
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require markStaleUnits after implementation
+let markStaleUnits;
+
+let mongoServer;
+let client;
+let db;
+
+const UNITS_COLLECTION = 'units_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ markStaleUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('markStaleUnits', () => {
+  // ---------------------------------------------------------------
+  // 1. updateMany is called with correct filter
+  //    (unit_code $nin currentCodes AND available: true)
+  // ---------------------------------------------------------------
+  describe('filter criteria', () => {
+    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
+      const logger = createMockLogger();
+
+      // Seed units in the database
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'A101', available: true, price: 1200 },
+        { unit_code: 'A102', available: true, price: 1300 },
+        { unit_code: 'B201', available: true, price: 1500 }
+      ]);
+
+      // Current scrape found A101 and A102, but NOT B201
+      const currentCodes = new Set(['A101', 'A102']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // B201 should be marked stale (it was available but not in current scrape)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify B201 was updated
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
+      expect(staleUnit.available).toBe(false);
+      expect(staleUnit.marked_stale_date).toBe(date);
+    });
+
+    it('should only affect units NOT in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'X1', available: true },
+        { unit_code: 'X2', available: true },
+        { unit_code: 'X3', available: true },
+        { unit_code: 'X4', available: true }
+      ]);
+
+      // Current scrape found X1 and X3
+      const currentCodes = new Set(['X1', 'X3']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // X1 and X3 should still be available
+      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
+      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
+      expect(x1.available).toBe(true);
+      expect(x3.available).toBe(true);
+
+      // X2 and X4 should be marked stale
+      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
+      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
+      expect(x2.available).toBe(false);
+      expect(x4.available).toBe(false);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. $set updates available to false and sets marked_stale_date
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set available to false for stale units', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'S100', available: true, price: 1000 },
+        { unit_code: 'S200', available: true, price: 1100 }
+      ]);
+
+      const currentCodes = new Set(['S100']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
+      expect(staleUnit.available).toBe(false);
+    });
+
+    it('should set marked_stale_date to the provided date', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'D100', available: true }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
+      expect(doc.marked_stale_date).toBe('2026-02-05');
+    });
+
+    it('should update both available and marked_stale_date in the same operation', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'C100', available: true, price: 900 }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-01-15';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
+      expect(doc.available).toBe(false);
+      expect(doc.marked_stale_date).toBe('2026-01-15');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Units in currentCodes are NOT marked stale
+  // ---------------------------------------------------------------
+  describe('units in currentCodes not affected', () => {
+    it('should not modify units that are in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'KEEP-1', available: true, price: 1200 },
+        { unit_code: 'KEEP-2', available: true, price: 1300 },
+        { unit_code: 'STALE-1', available: true, price: 1400 }
+      ]);
+
+      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // Kept units should retain their original state
+      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
+      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
+
+      expect(keep1.available).toBe(true);
+      expect(keep1.marked_stale_date).toBeUndefined();
+      expect(keep2.available).toBe(true);
+      expect(keep2.marked_stale_date).toBeUndefined();
+    });
+
+    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ALL-1', available: true },
+        { unit_code: 'ALL-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['ALL-1', 'ALL-2']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Already unavailable units are not modified
+  // ---------------------------------------------------------------
+  describe('already unavailable units', () => {
+    it('should not modify units that are already unavailable (available: false)', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
+        { unit_code: 'UNAVAIL-2', available: false },
+        { unit_code: 'AVAIL-1', available: true }
+      ]);
+
+      // None of these units are in the current scrape
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify the already-unavailable unit's date was NOT changed
+      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
+      expect(unavail1.marked_stale_date).toBe('2026-01-01');
+    });
+
+    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'U1', available: false },
+        { unit_code: 'U2', available: false }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Error handling when updateMany fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      try {
+        await markStaleUnits(mockDb, currentCodes, date, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to mark stale units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging of modified count
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of stale units marked', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'LOG-1', available: true },
+        { unit_code: 'LOG-2', available: true },
+        { unit_code: 'LOG-3', available: true }
+      ]);
+
+      const currentCodes = new Set(['LOG-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 2 })
+      );
+    });
+
+    it('should log count of 0 when no units are marked stale', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ZERO-1', available: true }
+      ]);
+
+      const currentCodes = new Set(['ZERO-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Empty currentUnitCodes handles gracefully
+  // ---------------------------------------------------------------
+  describe('empty currentUnitCodes', () => {
+    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'EMPTY-1', available: true },
+        { unit_code: 'EMPTY-2', available: true },
+        { unit_code: 'EMPTY-3', available: true }
+      ]);
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(3);
+
+      // All should be marked unavailable
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      for (const doc of docs) {
+        expect(doc.available).toBe(false);
+        expect(doc.marked_stale_date).toBe('2026-02-05');
+      }
+    });
+
+    it('should not throw when currentUnitCodes is an empty Set', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      // Should not throw even with empty collection and empty set
+      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
+    });
+
+    it('should handle empty database with empty currentUnitCodes', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the MongoDB updateMany result', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'RET-1', available: true },
+        { unit_code: 'RET-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['RET-1']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // updateMany result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('acknowledged');
+      expect(result.acknowledged).toBe(true);
+    });
+  });
+});

__tests__/scraper/parseUnits.test.js

@@ -0,0 +1,431 @@
+/**
+ * Tests for parseUnits() - HTML parsing with cheerio
+ *
+ * Covers:
+ * - Extraction of all data-spaces-* attributes
+ * - Image URL extraction from nested img elements
+ * - Container detection (section.spaces__tab-unit)
+ * - Empty / missing container handling
+ * - Deduplication by unit_code
+ * - Logging behavior (info, warn, error)
+ * - Missing attribute handling
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseUnits } = require('../../services/scraperService');
+
+const fixturesDir = path.join(__dirname, 'fixtures');
+
+// Helper: create a mock logger
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+describe('parseUnits', () => {
+  let sampleHtml;
+  let mockLogger;
+
+  beforeAll(() => {
+    sampleHtml = fs.readFileSync(
+      path.join(fixturesDir, 'sample-listing.html'),
+      'utf-8'
+    );
+  });
+
+  beforeEach(() => {
+    mockLogger = createMockLogger();
+  });
+
+  // ---------------------------------------------------------------
+  // 1. Container detection
+  // ---------------------------------------------------------------
+  describe('container detection', () => {
+    it('should find section.spaces__tab-unit container in valid HTML', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+
+      // Should not log an error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+      expect(units.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array when section container is missing', () => {
+      const html = '<html><body><div>No units here</div></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+
+    it('should return empty array for empty HTML string', () => {
+      const units = parseUnits('', mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    it('should return empty array when container exists but has no articles', () => {
+      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container found, no error about container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Extraction of all data-spaces-* attributes
+  // ---------------------------------------------------------------
+  describe('attribute extraction', () => {
+    it('should extract all expected attributes from a fully populated article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'W2707');
+
+      expect(unit1).toBeDefined();
+
+      // Core identifiers
+      expect(unit1.id).toBe('154842');
+      expect(unit1.unit_code).toBe('W2707');
+      expect(unit1.unit_id).toBe('154842');
+
+      // Physical attributes
+      expect(unit1.floor).toBe('2755');
+      expect(unit1.area).toBe('1210');
+      expect(unit1.bed_count).toBe('2');
+      expect(unit1.bath_count).toBe('2');
+
+      // Pricing
+      expect(unit1.price).toBe('5230');
+
+      // Availability
+      expect(unit1.available).toBe('true');
+      expect(unit1.unavailable).toBe('false');
+      expect(unit1.soonest).toBe('2025-10-11');
+      expect(unit1.date_available).toBe('1760140800');
+
+      // Plan information
+      expect(unit1.plan_id).toBe('10270');
+      expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
+
+      // Property information
+      expect(unit1.obj_type).toBe('unit');
+      expect(unit1.community).toBe('Country Club Towers');
+      expect(unit1.asset).toBe('420');
+
+      // URLs
+      expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
+      expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
+
+      // Specials
+      expect(unit1.specials_content).toBe('');
+    });
+
+    it('should extract correct count of units from the fixture', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      expect(units).toHaveLength(10);
+    });
+
+    it('should return raw string values without type conversion', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'W2707');
+
+      // All values should be strings (raw extraction, no conversion)
+      expect(typeof unit.id).toBe('string');
+      expect(typeof unit.price).toBe('string');
+      expect(typeof unit.floor).toBe('string');
+      expect(typeof unit.available).toBe('string');
+      expect(typeof unit.bed_count).toBe('string');
+      expect(typeof unit.bath_count).toBe('string');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Image URL extraction
+  // ---------------------------------------------------------------
+  describe('image URL extraction', () => {
+    it('should extract image_url from img src attribute', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
+            <img src="https://example.com/images/unit-1.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
+    });
+
+    it('should fall back to data-src when src is not present', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
+            <img data-src="https://example.com/images/unit-2.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
+    });
+
+    it('should not set image_url when no img element exists', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
+            <div>No image here</div>
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Missing attributes
+  // ---------------------------------------------------------------
+  describe('missing attributes', () => {
+    it('should return undefined for attributes not present on the article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit).toBeDefined();
+      expect(unit.floor).toBeUndefined();
+      expect(unit.area).toBeUndefined();
+      expect(unit.bed_count).toBeUndefined();
+      expect(unit.bath_count).toBeUndefined();
+      expect(unit.soonest).toBeUndefined();
+      expect(unit.date_available).toBeUndefined();
+      expect(unit.plan_id).toBeUndefined();
+      expect(unit.plan_name).toBeUndefined();
+      expect(unit.href).toBeUndefined();
+      expect(unit.inventory_href).toBeUndefined();
+      expect(unit.specials_content).toBeUndefined();
+    });
+
+    it('should still extract the attributes that are present on a sparse article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.id).toBe('1005');
+      expect(unit.unit_code).toBe('SPARSE-1');
+      expect(unit.unit_id).toBe('5005');
+      expect(unit.price).toBe('1300');
+      expect(unit.available).toBe('true');
+      expect(unit.obj_type).toBe('unit');
+      expect(unit.community).toBe('Country Club Towers');
+      expect(unit.asset).toBe('100');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Deduplication
+  // ---------------------------------------------------------------
+  describe('deduplication', () => {
+    it('should deduplicate units with the same unit_code', () => {
+      // Build HTML with duplicate articles
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+      const codes = units.map(u => u.unit_code);
+
+      // Should have 2 unique unit codes
+      expect(units).toHaveLength(2);
+      expect(new Set(codes).size).toBe(codes.length);
+    });
+
+    it('should keep the first occurrence when deduplicating', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // First occurrence (id=1, price=1000) should be kept
+      expect(units[0].id).toBe('1');
+      expect(units[0].price).toBe('1000');
+    });
+
+    it('should log a warning when duplicates are removed', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Duplicate units removed',
+        expect.objectContaining({
+          original: 3,
+          deduplicated: 2
+        })
+      );
+    });
+
+    it('should not log a warning when there are no duplicates', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      // The fixture has no duplicates, so no warn about duplicates
+      const warnCalls = mockLogger.warn.mock.calls;
+      const duplicateWarns = warnCalls.filter(
+        call => call[0] === 'Duplicate units removed'
+      );
+      expect(duplicateWarns).toHaveLength(0);
+    });
+
+    it('should filter out articles with no unit_code during deduplication', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // Article without unit_code should be filtered out
+      expect(units).toHaveLength(1);
+      expect(units[0].unit_code).toBe('VALID-1');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging behavior
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of parsed units', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 10 })
+      );
+    });
+
+    it('should log error when container is not found', () => {
+      const html = '<html><body><p>Nothing here</p></body></html>';
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.error).toHaveBeenCalledTimes(1);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Multiple units with varied data
+  // ---------------------------------------------------------------
+  describe('varied unit data', () => {
+    it('should handle unavailable units correctly', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="4001"
+            data-spaces-unit="UNAVAIL-1"
+            data-spaces-sort-price="2000"
+            data-spaces-available="false"
+            data-spaces-unavailable="true"
+            data-spaces-soonest=""
+            data-spaces-sort-date="">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.available).toBe('false');
+      expect(unit.unavailable).toBe('true');
+      expect(unit.soonest).toBe('');
+      expect(unit.date_available).toBe('');
+    });
+
+    it('should handle half bath counts', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // E3205 is the penthouse with 2.5 baths
+      const unit = units.find(u => u.unit_code === 'E3205');
+
+      expect(unit.bath_count).toBe('2.5');
+    });
+
+    it('should handle studio units (bed_count = 0)', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // W2603 is the studio with bed_count 0
+      const unit = units.find(u => u.unit_code === 'W2603');
+
+      expect(unit.bed_count).toBe('0');
+    });
+  });
+});

__tests__/scraper/recordScraperRun.test.js

@@ -0,0 +1,307 @@
+/**
+ * Tests for recordScraperRun() history persistence
+ *
+ * Covers:
+ * - insertOne is called with runData plus recordedAt timestamp
+ * - All runData fields are preserved (jobId, trigger, status, duration, etc.)
+ * - Error is caught and logged but NOT thrown (graceful failure)
+ * - Returns null on error instead of crashing
+ * - Returns insert result on success
+ * - recordedAt is a valid Date object
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require after implementation
+let recordScraperRun;
+
+let mongoServer;
+let client;
+let db;
+let logger;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ recordScraperRun } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+
+  // Create fresh mock logger for each test
+  logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+});
+
+describe('recordScraperRun', () => {
+  const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
+
+  // Sample runData matching the structure from runScrape()'s finally block
+  function createSampleRunData(overrides = {}) {
+    return {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      dryRun: false,
+      status: 'success',
+      startedAt: '2026-02-06T06:00:00.000Z',
+      completedAt: '2026-02-06T06:00:12.345Z',
+      duration: 12345,
+      unitsProcessed: 50,
+      pricesInserted: 48,
+      newUnitsCount: 3,
+      rentedUnitsCount: 1,
+      staleUnitsCount: 2,
+      errors: [],
+      ...overrides
+    };
+  }
+
+  // ---------------------------------------------------------------
+  // 1. insertOne is called with runData plus recordedAt timestamp
+  // ---------------------------------------------------------------
+  describe('insertOne with runData and recordedAt', () => {
+    it('should insert a document into the scraper_runs collection', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+
+    it('should include a recordedAt field in the inserted document', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. All runData fields are preserved
+  // ---------------------------------------------------------------
+  describe('preserving all runData fields', () => {
+    it('should preserve jobId, trigger, status, and duration', async () => {
+      const runData = createSampleRunData({
+        jobId: 'preserve-test-001',
+        trigger: 'scheduled',
+        status: 'failed',
+        duration: 9999
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.jobId).toBe('preserve-test-001');
+      expect(doc.trigger).toBe('scheduled');
+      expect(doc.status).toBe('failed');
+      expect(doc.duration).toBe(9999);
+    });
+
+    it('should preserve unit processing metrics', async () => {
+      const runData = createSampleRunData({
+        unitsProcessed: 42,
+        pricesInserted: 40,
+        newUnitsCount: 5,
+        rentedUnitsCount: 2,
+        staleUnitsCount: 3
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.unitsProcessed).toBe(42);
+      expect(doc.pricesInserted).toBe(40);
+      expect(doc.newUnitsCount).toBe(5);
+      expect(doc.rentedUnitsCount).toBe(2);
+      expect(doc.staleUnitsCount).toBe(3);
+    });
+
+    it('should preserve timing fields (startedAt, completedAt)', async () => {
+      const runData = createSampleRunData({
+        startedAt: '2026-02-06T10:00:00.000Z',
+        completedAt: '2026-02-06T10:00:15.500Z'
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
+      expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
+    });
+
+    it('should preserve dryRun flag', async () => {
+      const runData = createSampleRunData({ dryRun: true });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.dryRun).toBe(true);
+    });
+
+    it('should preserve errors array when it has entries', async () => {
+      const runData = createSampleRunData({
+        status: 'failed',
+        errors: ['Connection timeout', 'Retry exhausted']
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
+    });
+
+    it('should preserve empty errors array for successful runs', async () => {
+      const runData = createSampleRunData({ errors: [] });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Error is caught and logged but NOT thrown (graceful failure)
+  // ---------------------------------------------------------------
+  describe('graceful error handling', () => {
+    it('should NOT throw when insertOne fails', async () => {
+      const runData = createSampleRunData();
+
+      // Create a mock db that throws on insertOne
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      // This should NOT throw
+      await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
+    });
+
+    it('should log error message when insert fails', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await recordScraperRun(mockDb, runData, logger);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to record scraper run',
+        { errorMessage: 'Disk full' }
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Returns null on error instead of crashing
+  // ---------------------------------------------------------------
+  describe('return value on error', () => {
+    it('should return null when insertOne throws', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const result = await recordScraperRun(mockDb, runData, logger);
+      expect(result).toBeNull();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Returns insert result on success
+  // ---------------------------------------------------------------
+  describe('return value on success', () => {
+    it('should return the insertOne result object', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result).toBeDefined();
+      expect(result).not.toBeNull();
+    });
+
+    it('should return result with acknowledged property', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.acknowledged).toBe(true);
+    });
+
+    it('should return result with insertedId', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.insertedId).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. recordedAt is a valid Date object
+  // ---------------------------------------------------------------
+  describe('recordedAt field', () => {
+    it('should set recordedAt as a Date instance', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+
+    it('should set recordedAt close to current time', async () => {
+      const beforeTime = new Date();
+
+      const runData = createSampleRunData();
+      await recordScraperRun(db, runData, logger);
+
+      const afterTime = new Date();
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+      expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
+    });
+
+    it('should not overwrite any existing runData fields with recordedAt', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+
+      // Verify the original fields still exist alongside recordedAt
+      expect(doc.jobId).toBe(runData.jobId);
+      expect(doc.status).toBe(runData.status);
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+  });
+});

__tests__/scraper/scraperConfig.test.js

@@ -0,0 +1,192 @@
+/**
+ * Tests for config/scraper.js
+ * Tests scraper configuration constants and environment variable overrides
+ */
+
+describe('config/scraper', () => {
+  // Store original env values to restore after tests
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    // Clear all scraper-related env vars before each test
+    delete process.env.SCRAPER_SCHEDULE;
+    delete process.env.SCRAPER_TIMEZONE;
+    delete process.env.SCRAPER_ENABLED;
+    delete process.env.SCRAPER_TIMEOUT;
+    delete process.env.SCRAPER_UNITS_COLLECTION;
+    delete process.env.SCRAPER_PRICES_COLLECTION;
+    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
+    delete process.env.SCRAPER_RUNS_COLLECTION;
+
+    // Clear the require cache to reload config with fresh env
+    jest.resetModules();
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    process.env = originalEnv;
+  });
+
+  describe('TARGET_URL', () => {
+    test('should be set to the correct apartment listing URL', () => {
+      const config = require('../../config/scraper');
+      expect(config.TARGET_URL).toBe(
+        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
+      );
+    });
+  });
+
+  describe('SCRAPER_SCHEDULE', () => {
+    test('should default to "0 6 * * *" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
+    });
+  });
+
+  describe('SCRAPER_TIMEZONE', () => {
+    test('should default to "UTC" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+    });
+  });
+
+  describe('SCRAPER_ENABLED', () => {
+    test('should default to true when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be false when env var is set to "false"', () => {
+      process.env.SCRAPER_ENABLED = 'false';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(false);
+    });
+
+    test('should be true when env var is set to any other value', () => {
+      process.env.SCRAPER_ENABLED = 'true';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be true when env var is empty string', () => {
+      process.env.SCRAPER_ENABLED = '';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+  });
+
+  describe('SCRAPER_TIMEOUT', () => {
+    test('should default to 30000 when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(30000);
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEOUT = '60000';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(60000);
+    });
+
+    test('should parse string to integer', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
+      expect(config.SCRAPER_TIMEOUT).toBe(45000);
+    });
+  });
+
+  describe('USER_AGENT', () => {
+    test('should be a valid User-Agent string', () => {
+      const config = require('../../config/scraper');
+      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
+    });
+  });
+
+  describe('RETRY_CONFIG', () => {
+    test('should have maxRetries property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
+      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
+    });
+
+    test('should have baseDelay property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
+      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
+    });
+
+    test('should have timeout property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
+      expect(config.RETRY_CONFIG.timeout).toBe(30000);
+    });
+
+    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG.timeout).toBe(45000);
+    });
+  });
+
+  describe('COLLECTIONS', () => {
+    describe('default values', () => {
+      test('UNITS should default to "units_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
+      });
+
+      test('PRICES should default to "unit_prices_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
+      });
+
+      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
+      });
+
+      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
+      });
+    });
+
+    describe('env var overrides', () => {
+      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
+        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
+      });
+
+      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
+        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
+      });
+
+      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
+        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
+      });
+
+      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
+        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
+      });
+    });
+  });
+});

__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

__tests__/scraper/scraperService.test.js

@@ -0,0 +1,676 @@
+/**
+ * Tests for runScrape() orchestration function
+ *
+ * Covers:
+ * - Full workflow completes with mocked dependencies
+ * - Returns correct result structure with jobId, status, metrics
+ * - Handles dryRun option (skip DB writes)
+ * - Handles htmlContent option (use provided HTML)
+ * - Records scraper run to history on success
+ * - Records scraper run to history on failure
+ * - Catches and logs errors from fetchPage
+ * - Catches and logs errors from database operations
+ * - Calculates newUnitsCount and rentedUnitsCount correctly
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+let mongoServer;
+let client;
+let db;
+
+// We will require runScrape and recordScraperRun after implementation
+let runScrape;
+let recordScraperRun;
+
+// Store original module references so we can mock individual functions
+let scraperService;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  scraperService = require('../../services/scraperService');
+  ({ runScrape, recordScraperRun } = scraperService);
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all relevant collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+  // Reset all mocks
+  jest.restoreAllMocks();
+});
+
+// ============================================================
+// Helper: Create sample HTML with units
+// ============================================================
+function createSampleHtml(unitCodes) {
+  const articles = unitCodes.map(code => `
+    <article
+      data-spaces-id="100"
+      data-spaces-unit="${code}"
+      data-spaces-unit-id="200"
+      data-spaces-unit-floor="5"
+      data-spaces-sort-area="750"
+      data-spaces-sort-bed="1"
+      data-spaces-sort-bath="1"
+      data-spaces-sort-price="1500"
+      data-spaces-available="true"
+      data-spaces-unavailable="false"
+      data-spaces-soonest="Now"
+      data-spaces-sort-date="1700000000"
+      data-spaces-plan-id="10"
+      data-spaces-sort-plan-name="Studio"
+      data-spaces-obj="unit"
+      data-spaces-community="TestCommunity"
+      data-spaces-asset="1"
+      data-spaces-href="/unit/${code}"
+      data-spaces-inventory-href="/inventory/${code}"
+    >
+      <img src="http://example.com/${code}.jpg" />
+    </article>
+  `).join('\n');
+
+  return `
+    <html><body>
+      <section class="spaces__tab-unit">
+        ${articles}
+      </section>
+    </body></html>
+  `;
+}
+
+// ============================================================
+// Test: recordScraperRun
+// ============================================================
+describe('recordScraperRun', () => {
+  it('should insert a run record into scraper_runs collection', async () => {
+    const runData = {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      status: 'success',
+      startedAt: new Date().toISOString(),
+      completedAt: new Date().toISOString(),
+      duration: 1234,
+      unitsProcessed: 10,
+      pricesInserted: 8,
+      errors: []
+    };
+
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(db, runData, mockLogger);
+
+    expect(result).toBeTruthy();
+    expect(result.insertedId).toBeTruthy();
+
+    // Verify it was inserted
+    const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
+    expect(saved).toBeTruthy();
+    expect(saved.status).toBe('success');
+    expect(saved.recordedAt).toBeInstanceOf(Date);
+  });
+
+  it('should not throw when insert fails', async () => {
+    // Pass null db to cause an error
+    const runData = { jobId: 'test-fail', status: 'success' };
+
+    // Should not throw
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(null, runData, mockLogger);
+    expect(result).toBeNull();
+    expect(mockLogger.error).toHaveBeenCalled();
+  });
+});
+
+// ============================================================
+// Test: runScrape - Full workflow
+// ============================================================
+describe('runScrape', () => {
+  it('should complete full workflow with mocked dependencies', async () => {
+    const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
+
+    const result = await runScrape(db, {
+      trigger: 'manual',
+      jobId: 'test-full-workflow',
+      htmlContent: html
+    });
+
+    expect(result).toBeTruthy();
+    expect(result.jobId).toBe('test-full-workflow');
+    expect(result.status).toBe('success');
+    expect(result.trigger).toBe('manual');
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.errors).toEqual([]);
+  });
+
+  it('should return correct result structure with jobId, status, metrics', async () => {
+    const html = createSampleHtml(['UNIT-X']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-structure',
+      htmlContent: html
+    });
+
+    // Verify all expected fields exist
+    expect(result).toHaveProperty('jobId', 'test-structure');
+    expect(result).toHaveProperty('trigger', 'manual');
+    expect(result).toHaveProperty('dryRun', false);
+    expect(result).toHaveProperty('status', 'success');
+    expect(result).toHaveProperty('startedAt');
+    expect(result).toHaveProperty('completedAt');
+    expect(result).toHaveProperty('duration');
+    expect(result).toHaveProperty('unitsProcessed');
+    expect(result).toHaveProperty('pricesInserted');
+    expect(result).toHaveProperty('newUnitsCount');
+    expect(result).toHaveProperty('rentedUnitsCount');
+    expect(result).toHaveProperty('staleUnitsCount');
+    expect(result).toHaveProperty('errors');
+
+    // Verify types
+    expect(typeof result.duration).toBe('number');
+    expect(result.duration).toBeGreaterThanOrEqual(0);
+    expect(typeof result.startedAt).toBe('string');
+    expect(typeof result.completedAt).toBe('string');
+    expect(Array.isArray(result.errors)).toBe(true);
+  });
+
+  it('should generate a jobId when not provided', async () => {
+    const html = createSampleHtml(['UNIT-GEN']);
+
+    const result = await runScrape(db, {
+      htmlContent: html
+    });
+
+    expect(result.jobId).toBeTruthy();
+    expect(typeof result.jobId).toBe('string');
+    // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+    expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
+  });
+
+  // ============================================================
+  // Test: dryRun option
+  // ============================================================
+  it('should skip DB writes when dryRun is true', async () => {
+    const html = createSampleHtml(['DRY-A', 'DRY-B']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-dry-run',
+      htmlContent: html,
+      dryRun: true
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.dryRun).toBe(true);
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.pricesInserted).toBe(0);
+    expect(result.staleUnitsCount).toBe(0);
+
+    // Verify no units were written to the units collection
+    const unitsCount = await db.collection('units_migration_test').countDocuments();
+    expect(unitsCount).toBe(0);
+
+    // Verify no prices were written
+    const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
+    expect(pricesCount).toBe(0);
+  });
+
+  // ============================================================
+  // Test: htmlContent option
+  // ============================================================
+  it('should use provided HTML instead of fetching when htmlContent is given', async () => {
+    const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
+
+    // Mock axios.get to track if HTTP fetch is attempted
+    const axios = require('axios');
+    const axiosSpy = jest.spyOn(axios, 'get');
+
+    const result = await runScrape(db, {
+      jobId: 'test-html-content',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(3);
+    // axios.get should NOT have been called since we provided htmlContent
+    expect(axiosSpy).not.toHaveBeenCalled();
+  });
+
+  // ============================================================
+  // Test: Records scraper run on success
+  // ============================================================
+  it('should record scraper run to history on success', async () => {
+    const html = createSampleHtml(['REC-A']);
+
+    await runScrape(db, {
+      jobId: 'test-record-success',
+      htmlContent: html
+    });
+
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('success');
+    expect(runRecord.recordedAt).toBeInstanceOf(Date);
+    expect(runRecord.unitsProcessed).toBe(1);
+  });
+
+  // ============================================================
+  // Test: Records scraper run on failure
+  // ============================================================
+  it('should record scraper run to history on failure', async () => {
+    const html = createSampleHtml(['FAIL-A']);
+
+    // Create a db proxy that throws on bulkWrite (used by upsertUnits)
+    // but allows other operations (like scraper_runs insertOne) to pass through
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'units_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Database connection lost');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-record-failure',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Database connection lost');
+
+    // The run should still be recorded (via the real db passed through for scraper_runs)
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+    expect(runRecord.errors).toContain('Database connection lost');
+  });
+
+  // ============================================================
+  // Test: Catches errors from fetchPage
+  // ============================================================
+  it('should catch and handle errors from fetchPage', async () => {
+    // Mock axios.get to throw a network error (fetchPage uses axios internally)
+    const axios = require('axios');
+    jest.spyOn(axios, 'get').mockRejectedValue(
+      new Error('Network timeout')
+    );
+
+    const result = await runScrape(db, {
+      jobId: 'test-fetch-error'
+      // No htmlContent, so it will call fetchPage which uses axios
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Network timeout');
+    expect(result.unitsProcessed).toBe(0);
+  });
+
+  // ============================================================
+  // Test: Catches errors from database operations
+  // ============================================================
+  it('should catch and handle errors from database operations', async () => {
+    const html = createSampleHtml(['DB-ERR']);
+
+    // Create a db proxy that throws on the prices collection bulkWrite
+    // Use Proxy to properly delegate all methods to the real collection
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'unit_prices_migration_test') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Write concern timeout');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-db-error',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Write concern timeout');
+
+    // Run should still be recorded despite error
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+  });
+
+  // ============================================================
+  // Test: Calculates newUnitsCount and rentedUnitsCount correctly
+  // ============================================================
+  it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
+    // First, simulate yesterday's data by inserting price records for yesterday
+    const today = new Date();
+    const yesterday = new Date(today);
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const yesterdayStr = yesterday.toISOString().split('T')[0];
+
+    // Yesterday had units: OLD-A, OLD-B, OLD-C
+    await db.collection('unit_prices_migration_test').insertMany([
+      { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
+      { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
+      { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
+    ]);
+
+    // Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
+    const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-calc-changes',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    // NEW-D is new (not in yesterday's data)
+    expect(result.newUnitsCount).toBe(1);
+    // OLD-C was in yesterday's data but not in today's scrape
+    expect(result.rentedUnitsCount).toBe(1);
+    expect(result.unitsProcessed).toBe(3);
+  });
+
+  // ============================================================
+  // Test: Default trigger is 'manual'
+  // ============================================================
+  it('should default trigger to manual', async () => {
+    const html = createSampleHtml(['DEF-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-default-trigger',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('manual');
+  });
+
+  // ============================================================
+  // Test: Supports scheduled trigger
+  // ============================================================
+  it('should support scheduled trigger', async () => {
+    const html = createSampleHtml(['SCHED-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-scheduled-trigger',
+      trigger: 'scheduled',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('scheduled');
+  });
+
+  // ============================================================
+  // Test: Handles empty HTML (no units found)
+  // ============================================================
+  it('should handle HTML with no units gracefully', async () => {
+    const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+
+    const result = await runScrape(db, {
+      jobId: 'test-empty-html',
+      htmlContent: emptyHtml
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(0);
+    expect(result.errors).toContain('No units found in HTML');
+  });
+});
+
+// ============================================================
+// Test: sanitizeError - Error message sanitization
+// ============================================================
+describe('sanitizeError', () => {
+  let sanitizeError;
+
+  beforeAll(() => {
+    ({ sanitizeError } = require('../../services/scraperService'));
+  });
+
+  // ----------------------------------------------------------
+  // File path sanitization
+  // ----------------------------------------------------------
+  describe('file path sanitization', () => {
+    it('should remove Unix absolute file paths from error messages', () => {
+      const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/home\/user/);
+      expect(sanitized.message).toContain('ENOENT');
+    });
+
+    it('should remove /var paths from error messages', () => {
+      const error = new Error('Failed to read /var/app/current/data/secrets.yml');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/var\/app/);
+      expect(sanitized.message).toContain('Failed to read');
+    });
+
+    it('should remove Windows-style file paths from error messages', () => {
+      const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/C:\\Users/);
+      expect(sanitized.message).toContain('Cannot find module');
+    });
+
+    it('should remove /tmp and /usr paths from error messages', () => {
+      const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/tmp\//);
+      expect(sanitized.message).not.toMatch(/\/usr\//);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // MongoDB connection string sanitization
+  // ----------------------------------------------------------
+  describe('connection string sanitization', () => {
+    it('should redact mongodb:// connection strings', () => {
+      const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('s3cretP4ss');
+      expect(sanitized.message).not.toContain('admin:');
+      expect(sanitized.message).toContain('Connection failed');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact mongodb+srv:// connection strings', () => {
+      const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('password123');
+      expect(sanitized.message).not.toContain('user:');
+      expect(sanitized.message).toContain('Timeout connecting to');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact connection string without credentials', () => {
+      const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Credential / secret sanitization
+  // ----------------------------------------------------------
+  describe('credential sanitization', () => {
+    it('should redact common environment variable patterns', () => {
+      const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('sk-abc123xyz');
+      expect(sanitized.message).not.toContain('bearer-9876');
+    });
+
+    it('should redact password patterns', () => {
+      const error = new Error('Auth failed with password=MyS3cret!');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('MyS3cret!');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Error type preservation
+  // ----------------------------------------------------------
+  describe('error type preservation', () => {
+    it('should preserve the error type/name', () => {
+      const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TypeError');
+    });
+
+    it('should preserve custom error names', () => {
+      const error = new Error('Timeout at /var/app/scraper.js:100');
+      error.name = 'TimeoutError';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TimeoutError');
+    });
+
+    it('should preserve error name for RangeError', () => {
+      const error = new RangeError('Maximum call stack size exceeded');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('RangeError');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // General description preserved for debugging
+  // ----------------------------------------------------------
+  describe('general description preservation', () => {
+    it('should preserve a useful general description', () => {
+      const error = new Error('Network timeout after 30000ms');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('Network timeout after 30000ms');
+    });
+
+    it('should preserve error description when no sensitive data present', () => {
+      const error = new Error('Request failed with status code 500');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toBe('Request failed with status code 500');
+    });
+
+    it('should return a useful message even after heavy sanitization', () => {
+      const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('ECONNREFUSED');
+      expect(sanitized.message.length).toBeGreaterThan(5);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Stack trace sanitization
+  // ----------------------------------------------------------
+  describe('stack trace removal', () => {
+    it('should remove file paths from stack traces', () => {
+      const error = new Error('Something failed');
+      error.stack = 'Error: Something failed\n    at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n    at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).not.toMatch(/\/home\/user/);
+      expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
+    });
+
+    it('should handle errors without stack trace', () => {
+      const error = new Error('No stack');
+      error.stack = undefined;
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).toBeUndefined();
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Integration: sanitization applied before recordScraperRun()
+  // ----------------------------------------------------------
+  describe('sanitization before recordScraperRun()', () => {
+    it('should store sanitized error message in scraper_runs on failure', async () => {
+      const html = createSampleHtml(['SANITIZE-A']);
+
+      // Create a db proxy that throws an error containing sensitive info
+      const sensitiveError = new Error(
+        'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
+      );
+      sensitiveError.name = 'MongoServerError';
+
+      const faultyDb = {
+        collection: (name) => {
+          const realCollection = db.collection(name);
+          if (name === 'units_migration_test') {
+            return new Proxy(realCollection, {
+              get(target, prop) {
+                if (prop === 'bulkWrite') {
+                  return async () => { throw sensitiveError; };
+                }
+                const value = target[prop];
+                if (typeof value === 'function') {
+                  return value.bind(target);
+                }
+                return value;
+              }
+            });
+          }
+          return realCollection;
+        }
+      };
+
+      const result = await runScrape(faultyDb, {
+        jobId: 'test-sanitized-error',
+        htmlContent: html
+      });
+
+      expect(result.status).toBe('failed');
+
+      // Verify the error stored in result.errors is sanitized
+      const errorMsg = result.errors[0];
+      expect(errorMsg).not.toContain('SuperSecret');
+      expect(errorMsg).not.toContain('admin:');
+      expect(errorMsg).not.toContain('/home/deploy/');
+      expect(errorMsg).toContain('MongoServerError');
+
+      // Verify the error stored in scraper_runs is sanitized
+      const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
+      expect(runRecord).toBeTruthy();
+      expect(runRecord.status).toBe('failed');
+      const storedError = runRecord.errors[0];
+      expect(storedError).not.toContain('SuperSecret');
+      expect(storedError).not.toContain('admin:');
+      expect(storedError).not.toContain('/home/deploy/');
+    });
+  });
+});

__tests__/scraper/updateDailySummary.test.js

@@ -0,0 +1,561 @@
+/**
+ * Tests for updateDailySummary() daily summary aggregation
+ *
+ * Covers:
+ * - Upsert by date key (YYYY-MM-DD)
+ * - Summary includes new_units, rented_units, stale_units arrays
+ * - Count fields are calculated correctly
+ * - net_change = newUnits.length - rentedUnits.length
+ * - total_available_today is set correctly
+ * - total_available_yesterday fetched from previous day
+ * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+ * - turnover_rate is 0 when yesterdayTotal is 0
+ * - Error handling when updateOne fails
+ * - Logging of summary update details
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require updateDailySummary after implementation
+let updateDailySummary;
+
+let mongoServer;
+let client;
+let db;
+
+const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ updateDailySummary } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('updateDailySummary', () => {
+  // ---------------------------------------------------------------
+  // 1. Upsert by date key
+  // ---------------------------------------------------------------
+  describe('upsert by date key', () => {
+    it('should insert a new summary when no document exists for the date', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 2,
+        totalAvailable: 15
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have upserted (created new)
+      expect(result.upsertedCount).toBe(1);
+
+      // Verify the document in the database
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc).not.toBeNull();
+      expect(doc.date).toBe('2026-02-05');
+    });
+
+    it('should update an existing summary for the same date (idempotent upsert)', async () => {
+      const logger = createMockLogger();
+
+      // Insert initial summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-05',
+        new_units: ['A101'],
+        rented_units: [],
+        total_available_today: 10
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 12
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have matched and modified (not upserted)
+      expect(result.matchedCount).toBe(1);
+
+      // Verify updated values
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
+      expect(doc.total_available_today).toBe(12);
+
+      // Verify only one document exists for this date
+      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Summary includes new_units, rented_units, stale_units arrays
+  // ---------------------------------------------------------------
+  describe('summary arrays', () => {
+    it('should store new_units, rented_units, and stale_units arrays', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202', 'B203'],
+        staleUnitsCount: 3,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102']);
+      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
+      expect(doc.stale_units).toEqual([]);
+    });
+
+    it('should handle empty arrays for new_units and rented_units', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 25
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual([]);
+      expect(doc.rented_units).toEqual([]);
+      expect(doc.stale_units).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Count fields are calculated correctly
+  // ---------------------------------------------------------------
+  describe('count fields', () => {
+    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 5,
+        totalAvailable: 30
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(3);
+      expect(doc.rented_units_count).toBe(1);
+      expect(doc.stale_units_count).toBe(5);
+    });
+
+    it('should set all count fields to 0 when arrays are empty', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(0);
+      expect(doc.rented_units_count).toBe(0);
+      expect(doc.stale_units_count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. net_change = newUnits.length - rentedUnits.length
+  // ---------------------------------------------------------------
+  describe('net_change calculation', () => {
+    it('should calculate positive net_change when more units added than rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(2); // 3 - 1 = 2
+    });
+
+    it('should calculate negative net_change when more units rented than added', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
+    });
+
+    it('should calculate zero net_change when equal new and rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. total_available_today is set correctly
+  // ---------------------------------------------------------------
+  describe('total_available_today', () => {
+    it('should store the totalAvailable value as total_available_today', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 42
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(42);
+    });
+
+    it('should handle totalAvailable of 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 0
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. total_available_yesterday fetched from previous day
+  // ---------------------------------------------------------------
+  describe('total_available_yesterday', () => {
+    it('should fetch total_available_today from previous day summary', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 35,
+        new_units: [],
+        rented_units: []
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 35
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(35);
+    });
+
+    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(0);
+    });
+
+    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
+      const logger = createMockLogger();
+
+      // Insert March 31 summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-03-31',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-04-01',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 48
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
+      expect(doc.total_available_yesterday).toBe(50);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+  // ---------------------------------------------------------------
+  describe('turnover_rate calculation', () => {
+    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 50 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
+        staleUnitsCount: 0,
+        totalAvailable: 46
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 5 rented / 50 yesterday = 10%
+      expect(doc.turnover_rate).toBe(10);
+    });
+
+    it('should round turnover_rate to 2 decimal places', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 30 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 30
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 28
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
+      expect(doc.turnover_rate).toBe(6.67);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. turnover_rate is 0 when yesterdayTotal is 0
+  // ---------------------------------------------------------------
+  describe('turnover_rate edge cases', () => {
+    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 0 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 0
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Error handling when updateOne fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw and log error when updateOne fails', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      // Create a mock db that throws on updateOne
+      const mockCollection = {
+        findOne: jest.fn().mockResolvedValue(null),
+        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(
+        updateDailySummary(mockDb, summaryData, logger)
+      ).rejects.toThrow('Write concern timeout');
+
+      // Verify error was logged
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to update daily summary',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Logging of summary update details
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log summary update details on success', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 18
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Daily summary updated',
+        expect.objectContaining({
+          date: '2026-02-05',
+          newUnits: 2,
+          rentedUnits: 1,
+          totalAvailable: 18
+        })
+      );
+    });
+
+    it('should include timestamp field in the stored summary', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.timestamp).toBeDefined();
+      expect(typeof doc.timestamp).toBe('string');
+      // Verify it is a valid ISO timestamp
+      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
+    });
+  });
+});

__tests__/scraper/upsertUnits.test.js

@@ -0,0 +1,498 @@
+/**
+ * Tests for upsertUnits() bulk operation
+ *
+ * Covers:
+ * - bulkWrite is called with correct updateOne operations
+ * - upsert: true is set for all operations
+ * - $set updates last_scraped and data_source fields
+ * - $setOnInsert sets first_seen for new units
+ * - Empty units array logs warning and returns early
+ * - ordered: false is set for parallel execution
+ * - Error handling when bulkWrite fails
+ * - Logging of matched, modified, and upserted counts
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// We will require upsertUnits after implementation
+let upsertUnits;
+
+let mongoServer;
+let client;
+let db;
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ upsertUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the units collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('upsertUnits', () => {
+  const UNITS_COLLECTION = 'units_migration_test';
+
+  // ---------------------------------------------------------------
+  // 1. bulkWrite is called with correct updateOne operations
+  // ---------------------------------------------------------------
+  describe('bulkWrite operations', () => {
+    it('should create updateOne operations for each unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
+        { unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.upsertedCount).toBe(2);
+
+      // Verify documents exist in the collection
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'B202']);
+    });
+
+    it('should filter by unit_code as the unique key', async () => {
+      const logger = createMockLogger();
+
+      // Insert a unit first
+      const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
+      await upsertUnits(db, units, logger);
+
+      // Upsert again with updated price
+      const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
+      const result = await upsertUnits(db, updatedUnits, logger);
+
+      // Should match and modify, not insert new
+      expect(result.matchedCount).toBe(1);
+      expect(result.upsertedCount).toBe(0);
+
+      // Verify only one document exists
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+      expect(docs[0].price).toBe(1400);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. upsert: true is set for all operations
+  // ---------------------------------------------------------------
+  describe('upsert behavior', () => {
+    it('should insert new units when they do not exist (upsert: true)', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'NEW-001', price: 1100 },
+        { unit_code: 'NEW-002', price: 1200 },
+        { unit_code: 'NEW-003', price: 1300 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(3);
+
+      // Verify all three are in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+
+    it('should update existing units without creating duplicates', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
+
+      // Second upsert with same unit_code
+      await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. $set updates last_scraped and data_source fields
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set last_scraped timestamp on every upsert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'T100', price: 1000 }];
+      await upsertUnits(db, units, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
+      expect(doc.last_scraped).toBeDefined();
+      expect(typeof doc.last_scraped).toBe('string');
+      // The last_scraped should be between before and after timestamps
+      expect(doc.last_scraped >= beforeTime).toBe(true);
+      expect(doc.last_scraped <= afterTime).toBe(true);
+    });
+
+    it('should set data_source to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'T200', price: 1100 }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should update last_scraped on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const firstScraped = doc1.last_scraped;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const secondScraped = doc2.last_scraped;
+
+      expect(secondScraped > firstScraped).toBe(true);
+    });
+
+    it('should spread all unit fields into the $set operation', async () => {
+      const logger = createMockLogger();
+      const units = [{
+        unit_code: 'T400',
+        price: 1500,
+        bed_count: 2,
+        bath_count: 1.5,
+        area: 900,
+        floor: 3,
+        plan_name: 'Luxury',
+        community: 'Tower A'
+      }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
+      expect(doc.price).toBe(1500);
+      expect(doc.bed_count).toBe(2);
+      expect(doc.bath_count).toBe(1.5);
+      expect(doc.area).toBe(900);
+      expect(doc.floor).toBe(3);
+      expect(doc.plan_name).toBe('Luxury');
+      expect(doc.community).toBe('Tower A');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. $setOnInsert sets first_seen for new units
+  // ---------------------------------------------------------------
+  describe('$setOnInsert for first_seen', () => {
+    it('should set first_seen on initial insert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
+      expect(doc.first_seen).toBeDefined();
+      expect(typeof doc.first_seen).toBe('string');
+      expect(doc.first_seen >= beforeTime).toBe(true);
+    });
+
+    it('should NOT update first_seen on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+      const originalFirstSeen = doc1.first_seen;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert (update)
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+
+      // first_seen should remain unchanged
+      expect(doc2.first_seen).toBe(originalFirstSeen);
+    });
+
+    it('should set different first_seen for different units inserted at different times', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
+
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
+
+      // Both should have first_seen, but different values
+      expect(doc1.first_seen).toBeDefined();
+      expect(doc2.first_seen).toBeDefined();
+      expect(doc2.first_seen > doc1.first_seen).toBe(true);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty units array logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty units array', () => {
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
+    });
+
+    it('should return early with zero counts for empty array', async () => {
+      const logger = createMockLogger();
+
+      const result = await upsertUnits(db, [], logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(result.upsertedCount).toBe(0);
+    });
+
+    it('should not perform any database writes for empty array', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. ordered: false is set for parallel execution
+  // ---------------------------------------------------------------
+  describe('ordered: false for parallel execution', () => {
+    it('should successfully process all units even if one has a conflict', async () => {
+      const logger = createMockLogger();
+
+      // Insert multiple units - ordered: false means all operations execute
+      // even if some fail. We verify by checking all valid units are present.
+      const units = [
+        { unit_code: 'P100', price: 1000 },
+        { unit_code: 'P200', price: 1100 },
+        { unit_code: 'P300', price: 1200 },
+        { unit_code: 'P400', price: 1300 },
+        { unit_code: 'P500', price: 1400 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(5);
+
+      // All 5 units should be in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+
+    it('should handle a mix of new inserts and existing updates', async () => {
+      const logger = createMockLogger();
+
+      // First, insert some units
+      await upsertUnits(db, [
+        { unit_code: 'M100', price: 900 },
+        { unit_code: 'M200', price: 1000 }
+      ], logger);
+
+      // Now upsert a mix of existing and new units
+      const mixedUnits = [
+        { unit_code: 'M100', price: 950 },  // existing - update
+        { unit_code: 'M200', price: 1050 }, // existing - update
+        { unit_code: 'M300', price: 1100 }, // new - insert
+      ];
+
+      const result = await upsertUnits(db, mixedUnits, logger);
+
+      expect(result.matchedCount).toBe(2);
+      expect(result.upsertedCount).toBe(1);
+
+      // Total should be 3 documents
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E100', price: 1000 }];
+
+      // Create a mock db that throws on bulkWrite
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E200', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await upsertUnits(mockDb, units, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to upsert units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E300', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Logging of matched, modified, and upserted counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log matched, modified, and upserted counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L100', price: 1000 },
+        { unit_code: 'L200', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: expect.any(Number),
+          modified: expect.any(Number),
+          upserted: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L300', price: 1000 },
+        { unit_code: 'L400', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      // For new inserts: matched=0, modified=0, upserted=2
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 0,
+          modified: 0,
+          upserted: 2
+        })
+      );
+    });
+
+    it('should log correct counts for updates to existing units', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
+
+      // Reset mock to capture only the second call
+      logger.info.mockClear();
+
+      // Update existing
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 1,
+          modified: 1,
+          upserted: 0
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the bulkWrite result object', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'R100', price: 1000 }];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // bulkWrite result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('upsertedCount');
+    });
+  });
+});

config/scraper.js

@@ -0,0 +1,40 @@
+/**
+ * Scraper Configuration Module
+ *
+ * Contains all constants and settings for the apartment price scraper.
+ * All values can be overridden via environment variables for deployment flexibility.
+ *
+ * See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
+ */
+
+module.exports = {
+  // Target URL for apartment listings
+  TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
+
+  // Scheduling configuration
+  SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
+  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
+  SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
+
+  // HTTP settings
+  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
+
+  // Retry configuration for HTTP requests
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
+    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
+  },
+
+  // Graceful shutdown timeout (how long to wait for running job before force-stopping)
+  SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
+
+  // MongoDB collection names (environment variable overrides for development isolation)
+  COLLECTIONS: {
+    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
+    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
+    DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
+    SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
+  }
+};

eslint.config.mjs

@@ -0,0 +1,34 @@
+import js from '@eslint/js';
+import globals from 'globals';
+
+export default [
+  js.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'commonjs',
+      globals: {
+        ...globals.node,
+        ...globals.jest,
+      },
+    },
+    rules: {
+      // Allow unused vars with underscore prefix, and unused function args
+      'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
+      'no-console': 'off',
+      // Formatting rules as warnings (won't fail CI, but will be reported)
+      'semi': ['warn', 'always'],
+      'no-multiple-empty-lines': ['warn', { max: 2 }],
+      'eol-last': ['warn', 'always'],
+      'no-trailing-spaces': 'warn',
+    },
+  },
+  {
+    ignores: [
+      'node_modules/**',
+      'coverage/**',
+      '*.min.js',
+      'eslint.config.mjs',
+    ],
+  },
+];

jobs/scraperJob.js

@@ -0,0 +1,275 @@
+const cron = require('node-cron');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { runScrape } = require('../services/scraperService');
+const { createLogger } = require('../services/scraperLogger');
+
+// In-process mutex state
+let isRunning = false;
+let currentJobId = null;
+
+// Scheduler state
+let scheduledJob = null;
+
+// Shutdown state
+let shuttingDown = false;
+let runningJobPromise = null;
+
+/**
+ * Check if scraper is currently running
+ * @returns {boolean}
+ */
+function isScraperRunning() {
+  return isRunning;
+}
+
+/**
+ * Get current job ID if running
+ * @returns {string|null}
+ */
+function getCurrentJobId() {
+  return currentJobId;
+}
+
+/**
+ * Acquire the scraper lock
+ * @param {string} jobId - Job ID to set
+ * @returns {boolean} True if lock acquired
+ */
+function acquireLock(jobId) {
+  if (isRunning || shuttingDown) {
+    return false;
+  }
+  isRunning = true;
+  currentJobId = jobId;
+  return true;
+}
+
+/**
+ * Release the scraper lock
+ */
+function releaseLock() {
+  isRunning = false;
+  currentJobId = null;
+}
+
+/**
+ * Get the configured schedule expression
+ * @returns {string} Cron expression or 'disabled'
+ */
+function getScheduleExpression() {
+  if (!config.SCRAPER_ENABLED) {
+    return 'disabled';
+  }
+  return config.SCRAPER_SCHEDULE;
+}
+
+/**
+ * Calculate next scheduled run time
+ * @returns {string|null} ISO timestamp or null if disabled
+ */
+function getNextScheduledRun() {
+  if (!config.SCRAPER_ENABLED || !scheduledJob) {
+    return null;
+  }
+
+  const { CronExpressionParser } = require('cron-parser');
+  try {
+    const interval = CronExpressionParser.parse(config.SCRAPER_SCHEDULE, {
+      tz: config.SCRAPER_TIMEZONE
+    });
+    return interval.next().toISOString();
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Check if schedule runs more frequently than 1 hour
+ * @param {string} schedule - Cron expression
+ * @returns {boolean} True if schedule is too frequent
+ */
+function isScheduleTooFrequent(schedule) {
+  const parts = schedule.trim().split(/\s+/);
+
+  if (parts.length < 5) return false;
+
+  const minuteField = parts[0];
+
+  // If minute field is */N with N < 60, it runs more than once per hour
+  if (/^\*\/\d+$/.test(minuteField)) {
+    const interval = parseInt(minuteField.substring(2), 10);
+    if (interval < 60) return true;
+  }
+
+  // If minute field is *, it runs every minute
+  if (minuteField === '*') return true;
+
+  return false;
+}
+
+/**
+ * Initialize the scraper scheduler
+ * @param {Db} db - MongoDB database instance
+ */
+function initializeScheduler(db) {
+  const logger = createLogger('scheduler');
+
+  // Check if scheduling is enabled
+  if (!config.SCRAPER_ENABLED) {
+    logger.info('Scraper scheduling is disabled');
+    return;
+  }
+
+  // Validate cron expression
+  if (!cron.validate(config.SCRAPER_SCHEDULE)) {
+    logger.error('Invalid cron schedule expression', {
+      schedule: config.SCRAPER_SCHEDULE
+    });
+    logger.warn('Falling back to default schedule: 0 6 * * *');
+    config.SCRAPER_SCHEDULE = '0 6 * * *';
+  }
+
+  // Validate minimum interval (1 hour)
+  if (isScheduleTooFrequent(config.SCRAPER_SCHEDULE)) {
+    logger.warn('Schedule interval less than 1 hour - adjusting to hourly', {
+      originalSchedule: config.SCRAPER_SCHEDULE
+    });
+    config.SCRAPER_SCHEDULE = '0 * * * *';
+  }
+
+  // Create the scheduled job
+  scheduledJob = cron.schedule(config.SCRAPER_SCHEDULE, async () => {
+    const jobId = crypto.randomUUID();
+    const jobLogger = createLogger(jobId);
+
+    jobLogger.info('Scheduled scrape triggered');
+
+    // Check if already running or shutting down
+    if (!acquireLock(jobId)) {
+      jobLogger.warn('Skipped - scrape already in progress');
+      return;
+    }
+
+    try {
+      const jobExecution = runScrape(db, { trigger: 'scheduled', jobId });
+      runningJobPromise = jobExecution;
+      await jobExecution;
+    } catch (error) {
+      jobLogger.error('Scheduled scrape failed', {
+        errorType: error.name,
+        errorMessage: error.message
+      });
+    } finally {
+      runningJobPromise = null;
+      releaseLock();
+    }
+  }, {
+    timezone: config.SCRAPER_TIMEZONE,
+    scheduled: true
+  });
+
+  logger.info('Scraper scheduler initialized', {
+    schedule: config.SCRAPER_SCHEDULE,
+    timezone: config.SCRAPER_TIMEZONE,
+    nextRun: getNextScheduledRun()
+  });
+}
+
+/**
+ * Stop the scheduler (for graceful shutdown)
+ */
+function stopScheduler() {
+  if (scheduledJob) {
+    scheduledJob.stop();
+    scheduledJob = null;
+  }
+}
+
+/**
+ * Check if the scraper is in the process of shutting down
+ * @returns {boolean}
+ */
+function isShuttingDown() {
+  return shuttingDown;
+}
+
+/**
+ * Perform a graceful shutdown of the scraper
+ * - Stops the cron scheduler to prevent new jobs
+ * - Waits for any running job to complete (with timeout)
+ * - Releases the mutex lock
+ * - Logs shutdown progress
+ * @returns {Promise<void>}
+ */
+async function gracefulShutdown() {
+  const logger = createLogger('shutdown');
+
+  logger.info('Shutdown initiated');
+
+  // Mark as shutting down to prevent new jobs
+  shuttingDown = true;
+
+  // Stop the cron scheduler
+  stopScheduler();
+
+  // Wait for running job to complete (with timeout)
+  if (isRunning && runningJobPromise) {
+    logger.info('Waiting for running job to complete', {
+      jobId: currentJobId,
+      timeout: config.SHUTDOWN_TIMEOUT
+    });
+
+    let timeoutHandle;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutHandle = setTimeout(() => resolve('timeout'), config.SHUTDOWN_TIMEOUT);
+    });
+
+    const result = await Promise.race([
+      runningJobPromise.then(() => 'completed').catch(() => 'completed'),
+      timeoutPromise
+    ]);
+
+    clearTimeout(timeoutHandle);
+
+    if (result === 'timeout') {
+      logger.warn('Shutdown wait for running job timed out', {
+        jobId: currentJobId,
+        timeout: config.SHUTDOWN_TIMEOUT
+      });
+    }
+  }
+
+  // Force release the lock
+  releaseLock();
+
+  logger.info('Shutdown complete');
+}
+
+/**
+ * Register process signal handlers for graceful shutdown
+ * Listens for SIGTERM and SIGINT signals
+ */
+function registerSignalHandlers() {
+  process.on('SIGTERM', () => {
+    gracefulShutdown();
+  });
+
+  process.on('SIGINT', () => {
+    gracefulShutdown();
+  });
+}
+
+module.exports = {
+  isScraperRunning,
+  getCurrentJobId,
+  acquireLock,
+  releaseLock,
+  getScheduleExpression,
+  getNextScheduledRun,
+  initializeScheduler,
+  stopScheduler,
+  isShuttingDown,
+  gracefulShutdown,
+  registerSignalHandlers
+};

package.json

@@ -8,7 +8,9 @@
         "dev": "nodemon server.js",
         "test": "jest",
         "test:watch": "jest --watch",
-        "test:coverage": "jest --coverage"
+        "test:coverage": "jest --coverage",
+        "lint": "eslint .",
+        "lint:fix": "eslint . --fix"
     },
     "keywords": [
         "apartments",
@@ -19,17 +21,24 @@
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
+        "axios": "^1.13.4",
+        "cheerio": "^1.2.0",
         "cookie-parser": "^1.4.7",
         "cors": "^2.8.5",
+        "cron-parser": "^5.5.0",
         "express": "^4.18.2",
         "express-rate-limit": "^7.1.5",
         "jsonwebtoken": "^9.0.3",
         "mongodb": "^6.3.0",
+        "node-cron": "^4.2.1",
         "passport": "^0.7.0",
         "passport-google-oauth20": "^2.0.0",
         "uuid": "^13.0.0"
     },
     "devDependencies": {
+        "@eslint/js": "^9.18.0",
+        "eslint": "^9.18.0",
+        "globals": "^15.14.0",
         "jest": "^30.2.0",
         "mongodb-memory-server": "^11.0.1",
         "nodemon": "^3.0.2",

routes/admin.js

@@ -1142,5 +1142,236 @@ router.patch('/settings', async (req, res) => {
   }
 });
 
+// ============================================================
+// Scraper Endpoints
+// ============================================================
+
+const crypto = require('crypto');
+const {
+  isScraperRunning,
+  acquireLock,
+  releaseLock,
+  getCurrentJobId,
+  getScheduleExpression,
+  getNextScheduledRun
+} = require('../jobs/scraperJob');
+const { runScrape } = require('../services/scraperService');
+const { createLogger } = require('../services/scraperLogger');
+const scraperConfig = require('../config/scraper');
+
+// Logger for scraper admin routes
+const scraperRouteLogger = createLogger('scraper-admin');
+
+// Rate limiting for manual scrape trigger (per-user, in-memory)
+const RATE_LIMIT_MAX_REQUESTS = 5;
+const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour in milliseconds
+const rateLimitStore = new Map();
+
+/**
+ * Check rate limit for a given user ID.
+ * Returns an object indicating whether the request is allowed.
+ *
+ * @param {string} userId - The user ID to check
+ * @returns {{ allowed: boolean, retryAfterSeconds: number|null }}
+ */
+function checkRateLimit(userId) {
+  const now = Date.now();
+  const userKey = userId.toString();
+
+  if (!rateLimitStore.has(userKey)) {
+    rateLimitStore.set(userKey, []);
+  }
+
+  const timestamps = rateLimitStore.get(userKey);
+
+  // Remove timestamps outside the current window
+  const windowStart = now - RATE_LIMIT_WINDOW_MS;
+  const validTimestamps = timestamps.filter(ts => ts > windowStart);
+  rateLimitStore.set(userKey, validTimestamps);
+
+  if (validTimestamps.length >= RATE_LIMIT_MAX_REQUESTS) {
+    // Calculate when the oldest request in the window will expire
+    const oldestTimestamp = validTimestamps[0];
+    const retryAfterMs = (oldestTimestamp + RATE_LIMIT_WINDOW_MS) - now;
+    const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
+    return { allowed: false, retryAfterSeconds };
+  }
+
+  // Record this request
+  validTimestamps.push(now);
+  return { allowed: true, retryAfterSeconds: null };
+}
+
+/**
+ * Reset the rate limiter (for testing)
+ */
+function resetRateLimiter() {
+  rateLimitStore.clear();
+}
+
+/**
+ * POST /api/admin/scraper/run
+ * Trigger a manual scrape
+ *
+ * Request body (optional):
+ *   - dryRun: boolean - Skip database writes for safe testing
+ *   - htmlContent: string - Use provided HTML instead of fetching (for testing/debugging)
+ */
+router.post('/scraper/run', async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+    const { dryRun = false, htmlContent = null } = req.body || {};
+
+    // Check rate limit (per-user, before mutex check)
+    const rateLimitResult = checkRateLimit(req.user._id);
+    if (!rateLimitResult.allowed) {
+      res.setHeader('Retry-After', rateLimitResult.retryAfterSeconds.toString());
+      return res.status(429).json({
+        error: 'Rate limit exceeded. Maximum 5 trigger requests per hour.'
+      });
+    }
+
+    // Check if scraper is already running
+    if (isScraperRunning()) {
+      return res.status(409).json({ error: 'Scrape already in progress' });
+    }
+
+    // Generate job ID and acquire lock
+    const jobId = crypto.randomUUID();
+    acquireLock(jobId);
+
+    // Log admin action
+    await logActivity(db, {
+      userId: req.user._id.toString(),
+      action: 'ADMIN_TRIGGER_SCRAPE',
+      metadata: { jobId, dryRun, usingProvidedHtml: !!htmlContent }
+    });
+
+    // Start scrape asynchronously (do not await - return 202 immediately)
+    runScrape(db, { trigger: 'manual', jobId, dryRun, htmlContent })
+      .catch((error) => {
+        scraperRouteLogger.error('Async scrape failed', {
+          errorType: error.name,
+          errorMessage: error.message
+        });
+      })
+      .finally(() => releaseLock());
+
+    // Return immediately with job ID
+    res.status(202).json({
+      data: {
+        jobId,
+        status: 'started',
+        dryRun,
+        message: dryRun ? 'Scrape job initiated (dry run - no DB writes)' : 'Scrape job initiated'
+      }
+    });
+
+  } catch (error) {
+    scraperRouteLogger.error('Error triggering scrape', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    res.status(500).json({ error: 'Failed to start scrape job' });
+  }
+});
+
+/**
+ * GET /api/admin/scraper/status
+ * Get current scraper status including running state, last run details,
+ * next scheduled run, and schedule expression.
+ */
+router.get('/scraper/status', async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+
+    // Get last run from history
+    const lastRunDoc = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
+      .findOne({}, { sort: { startedAt: -1 } });
+
+    const lastRun = lastRunDoc ? {
+      jobId: lastRunDoc.jobId,
+      timestamp: lastRunDoc.startedAt,
+      status: lastRunDoc.status,
+      duration: lastRunDoc.duration,
+      trigger: lastRunDoc.trigger,
+      unitsProcessed: lastRunDoc.unitsProcessed,
+      pricesInserted: lastRunDoc.pricesInserted,
+      errors: lastRunDoc.errors?.length > 0 ? lastRunDoc.errors : null
+    } : null;
+
+    res.json({
+      data: {
+        currentStatus: isScraperRunning() ? 'running' : 'idle',
+        runningJobId: isScraperRunning() ? getCurrentJobId() : null,
+        lastRun,
+        nextScheduledRun: getNextScheduledRun(),
+        schedule: getScheduleExpression()
+      }
+    });
+
+  } catch (error) {
+    scraperRouteLogger.error('Error fetching scraper status', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    res.status(503).json({ error: 'Service temporarily unavailable' });
+  }
+});
+
+/**
+ * GET /api/admin/scraper/history
+ * Get scraper run history with pagination
+ *
+ * Query params:
+ *   - limit: Number of records (1-100, default 30)
+ *   - offset: Number of records to skip (default 0)
+ */
+router.get('/scraper/history', async (req, res) => {
+  try {
+    const db = req.app.locals.db;
+
+    // Parse and validate pagination parameters
+    let limit = parseInt(req.query.limit);
+    let offset = parseInt(req.query.offset);
+
+    // Validate limit
+    if (req.query.limit !== undefined) {
+      if (isNaN(limit) || limit < 1) {
+        return res.status(400).json({ error: 'Limit must be between 1 and 100' });
+      }
+      limit = Math.min(limit, 100);
+    } else {
+      limit = 30;
+    }
+
+    // Validate offset
+    if (req.query.offset !== undefined) {
+      if (isNaN(offset) || offset < 0) {
+        return res.status(400).json({ error: 'Invalid offset parameter' });
+      }
+    } else {
+      offset = 0;
+    }
+
+    const history = await db.collection(scraperConfig.COLLECTIONS.SCRAPER_RUNS)
+      .find({})
+      .sort({ startedAt: -1 })
+      .skip(offset)
+      .limit(limit)
+      .toArray();
+
+    res.json({ data: history });
+
+  } catch (error) {
+    scraperRouteLogger.error('Error fetching scraper history', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    res.status(503).json({ error: 'Service temporarily unavailable' });
+  }
+});
+
 module.exports = router;
 module.exports.clearStatsCache = clearStatsCache;
+module.exports.resetRateLimiter = resetRateLimiter;

services/scraperLogger.js

@@ -0,0 +1,164 @@
+/**
+ * Scraper-specific structured JSON logger
+ * Produces one JSON object per line to stdout
+ */
+
+const LEVELS = {
+  info: 'info',
+  warn: 'warn',
+  error: 'error'
+};
+
+/**
+ * Create a logger instance scoped to a job ID
+ * @param {string} jobId - Job identifier for correlation
+ * @returns {Object} Logger object with info, warn, error methods
+ */
+function createLogger(jobId) {
+  /**
+   * Internal log function
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} context - Additional context data
+   */
+  const log = (level, message, context = {}) => {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      message: truncateMessage(message),
+      jobId,
+      context: sanitizeContext(context)
+    };
+
+    // Output as single-line JSON
+    console.log(JSON.stringify(entry));
+  };
+
+  return {
+    info: (message, context) => log(LEVELS.info, message, context),
+    warn: (message, context) => log(LEVELS.warn, message, context),
+    error: (message, context) => log(LEVELS.error, message, context)
+  };
+}
+
+/**
+ * Truncate message to prevent log bloat
+ * @param {string} message - Message to truncate
+ * @param {number} maxLength - Maximum length (default 1000)
+ * @returns {string} Truncated message
+ */
+function truncateMessage(message, maxLength = 1000) {
+  if (message === null || message === undefined) {
+    return '';
+  }
+  const str = String(message);
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.substring(0, maxLength) + '... [truncated]';
+}
+
+/**
+ * Recursively process an object to handle Buffers, circular references, and sensitive data
+ * @param {*} obj - Object to process
+ * @param {WeakSet} seen - Set of seen objects for circular reference detection
+ * @returns {*} Processed value
+ */
+function processValue(obj, seen = new WeakSet()) {
+  // Handle null/undefined
+  if (obj === null) {
+    return null;
+  }
+  if (obj === undefined) {
+    return null;
+  }
+
+  // Handle Buffer BEFORE checking for object (Buffer is an object)
+  if (Buffer.isBuffer(obj)) {
+    return `[Buffer: ${obj.length} bytes]`;
+  }
+
+  // Handle strings - check for MongoDB connection strings
+  if (typeof obj === 'string') {
+    if (/mongodb(\+srv)?:\/\//.test(obj)) {
+      return redactConnectionString(obj);
+    }
+    return obj;
+  }
+
+  // Handle primitives
+  if (typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Handle circular references
+  if (seen.has(obj)) {
+    return '[Circular]';
+  }
+  seen.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => processValue(item, seen));
+  }
+
+  // Handle plain objects
+  const result = {};
+  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
+
+  for (const key of Object.keys(obj)) {
+    // Check for sensitive keys
+    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
+      result[key] = '[REDACTED]';
+    } else {
+      result[key] = processValue(obj[key], seen);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sanitize context object for safe logging
+ * - Remove circular references
+ * - Redact sensitive data
+ * - Handle special types (Buffer, undefined)
+ * @param {Object} context - Context object
+ * @returns {Object} Sanitized context
+ */
+function sanitizeContext(context) {
+  if (!context || typeof context !== 'object') {
+    return {};
+  }
+
+  try {
+    return processValue(context);
+  } catch (error) {
+    // If sanitization fails, return empty context
+    return { sanitizationError: 'Failed to sanitize context' };
+  }
+}
+
+/**
+ * Redact credentials from MongoDB connection string
+ * @param {string} uri - Connection string
+ * @returns {string} Redacted string
+ */
+function redactConnectionString(uri) {
+  try {
+    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
+    return uri.replace(
+      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
+      'mongodb$1://[user]:[REDACTED]@'
+    );
+  } catch {
+    return '[REDACTED CONNECTION STRING]';
+  }
+}
+
+module.exports = {
+  createLogger,
+  LEVELS,
+  // Export internal functions for testing
+  redactConnectionString
+};

services/scraperService.js

@@ -0,0 +1,888 @@
+/**
+ * Scraper Service
+ *
+ * Core scraper logic including HTTP fetching, HTML parsing,
+ * data transformation, and database operations.
+ */
+
+const axios = require('axios');
+const cheerio = require('cheerio');
+const crypto = require('crypto');
+const config = require('../config/scraper');
+const { createLogger } = require('./scraperLogger');
+
+/**
+ * Sleep utility for retry delays
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Determine if an error is retryable
+ * @param {Error} error - Axios error
+ * @returns {boolean} True if should retry
+ */
+function isRetryableError(error) {
+  // Network errors (timeout, DNS, connection) don't have a response property
+  if (!error.response) {
+    return true;
+  }
+
+  const status = error.response.status;
+
+  // 5xx server errors are retryable
+  if (status >= 500) {
+    return true;
+  }
+
+  // 429 Too Many Requests - do not retry immediately
+  if (status === 429) {
+    return false;
+  }
+
+  // 4xx client errors - do not retry
+  return false;
+}
+
+/**
+ * Fetch page HTML with retry logic
+ * @param {string} url - Target URL
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<string>} HTML content
+ * @throws {Error} After all retries exhausted
+ */
+async function fetchPage(url, logger) {
+  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    try {
+      logger.info('Fetching page', { url, attempt });
+
+      const response = await axios.get(url, {
+        timeout,
+        headers: {
+          'User-Agent': config.USER_AGENT
+        },
+        maxRedirects: 5,
+        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
+      });
+
+      logger.info('Page fetched successfully', {
+        status: response.status,
+        contentLength: response.data.length
+      });
+
+      return response.data;
+
+    } catch (error) {
+      lastError = error;
+
+      // Determine if error is retryable
+      const isRetryable = isRetryableError(error);
+
+      logger.warn('Fetch attempt failed', {
+        attempt,
+        errorType: error.name,
+        errorMessage: error.message,
+        statusCode: error.response?.status,
+        isRetryable
+      });
+
+      // Don't retry non-retryable errors (4xx)
+      if (!isRetryable) {
+        throw error;
+      }
+
+      // Don't wait after last attempt
+      if (attempt <= maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+        logger.info('Waiting before retry', { delay });
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Parse unit data from HTML using cheerio
+ * @param {string} html - HTML content
+ * @param {Object} logger - Logger instance
+ * @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
+ */
+function parseUnits(html, logger) {
+  const $ = cheerio.load(html);
+  const units = [];
+
+  // Find the main container
+  const container = $('section.spaces__tab-unit');
+
+  if (container.length === 0) {
+    logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
+    return [];
+  }
+
+  // Extract each article element
+  container.find('article').each((index, article) => {
+    const $article = $(article);
+
+    const unit = {
+      // Core identifiers
+      id: $article.attr('data-spaces-id'),
+      unit_code: $article.attr('data-spaces-unit'),
+      unit_id: $article.attr('data-spaces-unit-id'),
+
+      // Physical attributes
+      floor: $article.attr('data-spaces-unit-floor'),
+      area: $article.attr('data-spaces-sort-area'),
+      bed_count: $article.attr('data-spaces-sort-bed'),
+      bath_count: $article.attr('data-spaces-sort-bath'),
+
+      // Pricing
+      price: $article.attr('data-spaces-sort-price'),
+
+      // Availability
+      available: $article.attr('data-spaces-available'),
+      unavailable: $article.attr('data-spaces-unavailable'),
+      soonest: $article.attr('data-spaces-soonest'),
+      date_available: $article.attr('data-spaces-sort-date'),
+
+      // Plan information
+      plan_id: $article.attr('data-spaces-plan-id'),
+      plan_name: $article.attr('data-spaces-sort-plan-name'),
+
+      // Property information
+      obj_type: $article.attr('data-spaces-obj'),
+      community: $article.attr('data-spaces-community'),
+      asset: $article.attr('data-spaces-asset'),
+
+      // URLs
+      href: $article.attr('data-spaces-href'),
+      inventory_href: $article.attr('data-spaces-inventory-href'),
+
+      // Specials
+      specials_content: $article.attr('data-spaces-specials-content')
+    };
+
+    // Extract image URL from nested element if present
+    const imgElement = $article.find('img').first();
+    if (imgElement.length > 0) {
+      unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
+    }
+
+    units.push(unit);
+  });
+
+  logger.info('Units parsed', { count: units.length });
+
+  // Deduplicate by unit_code (in case of duplicate articles)
+  const seen = new Set();
+  const deduplicated = units.filter(unit => {
+    if (!unit.unit_code || seen.has(unit.unit_code)) {
+      return false;
+    }
+    seen.add(unit.unit_code);
+    return true;
+  });
+
+  if (deduplicated.length < units.length) {
+    logger.warn('Duplicate units removed', {
+      original: units.length,
+      deduplicated: deduplicated.length
+    });
+  }
+
+  return deduplicated;
+}
+
+// ============================================================
+// Data Type Conversion Helpers
+// ============================================================
+
+/**
+ * Parse string to integer, return null for invalid
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed integer or null
+ */
+function parseInteger(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Parse string to positive integer, return null for 0 or invalid.
+ * Used for fields like area where 0 means "not available".
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed positive integer or null
+ */
+function parsePositiveInteger(value) {
+  const parsed = parseInteger(value);
+  return parsed === 0 ? null : parsed;
+}
+
+/**
+ * Parse value that could be integer or string identifier.
+ * Returns integer if cleanly parseable, otherwise trimmed string.
+ * @param {*} value - Value to parse
+ * @returns {number|string|null} Parsed integer, trimmed string, or null
+ */
+function parseIntegerOrString(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  // If it parses cleanly to an integer, return integer
+  if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
+    return parsed;
+  }
+  // Otherwise return as trimmed string
+  return String(value).trim();
+}
+
+/**
+ * Parse price, stripping non-numeric characters.
+ * Handles "$1,234" format and "Call for pricing" text.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed price or null
+ */
+function parsePrice(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+
+  // Check for "Call for pricing" or similar text
+  if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
+    return null;
+  }
+
+  // Strip non-numeric characters except decimal point
+  const cleaned = String(value).replace(/[^0-9.]/g, '');
+  const parsed = parseInt(cleaned, 10);
+
+  if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
+    return null;
+  }
+
+  // Negative prices are invalid
+  if (parsed < 0) {
+    return null;
+  }
+
+  return parsed;
+}
+
+/**
+ * Parse boolean from string.
+ * Handles "true"/"false"/"1"/"0" and actual boolean values.
+ * @param {*} value - Value to parse
+ * @returns {boolean|null} Parsed boolean or null
+ */
+function parseBoolean(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  const str = String(value).toLowerCase().trim();
+  if (str === 'true' || str === '1') return true;
+  if (str === 'false' || str === '0') return false;
+  return null;
+}
+
+/**
+ * Parse float value.
+ * Named parseFloatValue to avoid shadowing the global parseFloat.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed float or null
+ */
+function parseFloatValue(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Trim string, return null for empty.
+ * Converts non-string values to string before trimming.
+ * @param {*} value - Value to trim
+ * @returns {string|null} Trimmed string or null
+ */
+function trimString(value) {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  const trimmed = String(value).trim();
+  return trimmed === '' ? null : trimmed;
+}
+
+// ============================================================
+// Main Data Type Conversion Function
+// ============================================================
+
+/**
+ * Convert unit data types from strings to proper types.
+ * Applies the appropriate parser to each field based on its expected type.
+ * @param {Object} unit - Raw unit object with string values
+ * @returns {Object} Unit object with converted types
+ */
+function convertDataTypes(unit) {
+  return {
+    // Integer fields
+    id: parseInteger(unit.id),
+    unit_id: parseIntegerOrString(unit.unit_id),
+    floor: parseInteger(unit.floor),
+    area: parsePositiveInteger(unit.area),
+    bed_count: parseInteger(unit.bed_count),
+    plan_id: parseInteger(unit.plan_id),
+    asset: parseInteger(unit.asset),
+    date_available: parseInteger(unit.date_available),
+
+    // Float fields
+    bath_count: parseFloatValue(unit.bath_count),
+
+    // Price field (special handling)
+    price: parsePrice(unit.price),
+
+    // Boolean fields
+    available: parseBoolean(unit.available),
+    unavailable: parseBoolean(unit.unavailable),
+
+    // String fields (trim and preserve)
+    unit_code: trimString(unit.unit_code),
+    plan_name: trimString(unit.plan_name),
+    soonest: trimString(unit.soonest),
+    obj_type: trimString(unit.obj_type),
+    community: trimString(unit.community),
+    href: trimString(unit.href),
+    inventory_href: trimString(unit.inventory_href),
+    image_url: trimString(unit.image_url),
+    specials_content: trimString(unit.specials_content)
+  };
+}
+
+// ============================================================
+// Date Helpers
+// ============================================================
+
+/**
+ * Calculate yesterday's date from a given date string.
+ * @param {string} dateStr - Date in YYYY-MM-DD format
+ * @returns {string} Yesterday's date in YYYY-MM-DD format
+ */
+function getYesterday(dateStr) {
+  const date = new Date(dateStr + 'T00:00:00Z');
+  date.setUTCDate(date.getUTCDate() - 1);
+  return date.toISOString().split('T')[0];
+}
+
+// ============================================================
+// Database Operations
+// ============================================================
+
+/**
+ * Upsert unit records to database using bulkWrite.
+ * Each unit is matched by unit_code as the unique key.
+ * Sets last_scraped and data_source on every update.
+ * Sets first_seen only on initial insert via $setOnInsert.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Bulk write result
+ */
+async function upsertUnits(db, units, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+  const now = new Date().toISOString();
+
+  const operations = units.map(unit => ({
+    updateOne: {
+      filter: { unit_code: unit.unit_code },
+      update: {
+        $set: {
+          ...unit,
+          last_scraped: now,
+          data_source: 'web_scraper'
+        },
+        $setOnInsert: {
+          first_seen: now
+        }
+      },
+      upsert: true
+    }
+  }));
+
+  if (operations.length === 0) {
+    logger.warn('No units to upsert');
+    return { modifiedCount: 0, upsertedCount: 0 };
+  }
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Units upserted', {
+      matched: result.matchedCount,
+      modified: result.modifiedCount,
+      upserted: result.upsertedCount
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to upsert units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Insert price records for today using bulkWrite with upsert.
+ * Uses unit_code + date_checked as the composite key to prevent
+ * duplicate price records for the same unit on the same day.
+ * Re-running on the same day updates existing records (idempotent).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Array<Object>} units - Array of unit objects
+ * @param {string} date - Date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Insert result with insertedCount
+ */
+async function insertPrices(db, units, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+  const now = new Date().toISOString();
+
+  // Filter out units without prices
+  const unitsWithPrices = units.filter(u => u.price !== null);
+
+  const priceRecords = unitsWithPrices.map(unit => ({
+    unit_code: unit.unit_code,
+    date_checked: date,
+    price: unit.price,
+    last_updated: now,
+    data_source: 'web_scraper'
+  }));
+
+  if (priceRecords.length === 0) {
+    logger.warn('No price records to insert');
+    return { insertedCount: 0 };
+  }
+
+  // Use updateOne with upsert to handle re-runs on same day
+  const operations = priceRecords.map(record => ({
+    updateOne: {
+      filter: {
+        unit_code: record.unit_code,
+        date_checked: record.date_checked
+      },
+      update: { $set: record },
+      upsert: true
+    }
+  }));
+
+  try {
+    const result = await collection.bulkWrite(operations, { ordered: false });
+
+    logger.info('Prices inserted', {
+      inserted: result.upsertedCount,
+      updated: result.modifiedCount
+    });
+
+    return { insertedCount: result.upsertedCount + result.modifiedCount };
+
+  } catch (error) {
+    logger.error('Failed to insert prices', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Mark units not in current scrape as stale/unavailable.
+ * Uses updateMany to set available: false and marked_stale_date
+ * for all units whose unit_code is NOT in the current scrape
+ * and that are currently available.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Set<string>} currentUnitCodes - Unit codes from current scrape
+ * @param {string} date - Current date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Update result
+ */
+async function markStaleUnits(db, currentUnitCodes, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+
+  try {
+    const result = await collection.updateMany(
+      {
+        unit_code: { $nin: [...currentUnitCodes] },
+        available: true
+      },
+      {
+        $set: {
+          available: false,
+          marked_stale_date: date
+        }
+      }
+    );
+
+    logger.info('Stale units marked', { count: result.modifiedCount });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to mark stale units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+/**
+ * Update daily summary document.
+ * Upserts by date field (YYYY-MM-DD format).
+ * Fetches yesterday's summary for comparison metrics.
+ * Calculates turnover_rate as (rentedUnits / yesterdayTotal * 100).
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} summaryData - Summary data
+ * @param {string} summaryData.date - Date in YYYY-MM-DD format
+ * @param {Array<string>} summaryData.newUnits - Unit codes added today
+ * @param {Array<string>} summaryData.rentedUnits - Unit codes removed today
+ * @param {number} summaryData.staleUnitsCount - Count of stale units
+ * @param {number} summaryData.totalAvailable - Total available units today
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} MongoDB updateOne result
+ */
+async function updateDailySummary(db, summaryData, logger) {
+  const collection = db.collection(config.COLLECTIONS.DAILY_SUMMARIES);
+  const { date, newUnits, rentedUnits, staleUnitsCount, totalAvailable } = summaryData;
+
+  // Calculate yesterday's date for comparison
+  const yesterday = getYesterday(date);
+
+  // Get yesterday's summary for comparison
+  const yesterdaySummary = await collection.findOne({ date: yesterday });
+  const yesterdayTotal = yesterdaySummary?.total_available_today || 0;
+
+  const summary = {
+    date,
+    timestamp: new Date().toISOString(),
+    new_units: newUnits,
+    rented_units: rentedUnits,
+    stale_units: [], // Stale units list is not tracked per PRD
+    new_units_count: newUnits.length,
+    rented_units_count: rentedUnits.length,
+    stale_units_count: staleUnitsCount,
+    net_change: newUnits.length - rentedUnits.length,
+    total_available_today: totalAvailable,
+    total_available_yesterday: yesterdayTotal,
+    turnover_rate: yesterdayTotal > 0
+      ? Math.round((rentedUnits.length / yesterdayTotal) * 10000) / 100
+      : 0
+  };
+
+  try {
+    const result = await collection.updateOne(
+      { date },
+      { $set: summary },
+      { upsert: true }
+    );
+
+    logger.info('Daily summary updated', {
+      date,
+      newUnits: newUnits.length,
+      rentedUnits: rentedUnits.length,
+      totalAvailable
+    });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to update daily summary', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
+// ============================================================
+// Additional Helpers for runScrape Orchestration
+// ============================================================
+
+/**
+ * Get today's date in YYYY-MM-DD format (UTC).
+ * @returns {string} Today's date string
+ */
+function getTodayUTC() {
+  return new Date().toISOString().split('T')[0];
+}
+
+/**
+ * Get the set of unit codes that had price records yesterday.
+ * Used to calculate new and rented units by comparison.
+ * @param {Db} db - MongoDB database instance
+ * @param {string} today - Today's date in YYYY-MM-DD format
+ * @returns {Promise<Set<string>>} Set of unit codes from yesterday
+ */
+async function getYesterdayUnitCodes(db, today) {
+  const yesterday = getYesterday(today);
+  const collection = db.collection(config.COLLECTIONS.PRICES);
+
+  const yesterdayRecords = await collection
+    .find({ date_checked: yesterday }, { projection: { unit_code: 1 } })
+    .toArray();
+
+  return new Set(yesterdayRecords.map(r => r.unit_code));
+}
+
+// ============================================================
+// Error Sanitization
+// ============================================================
+
+/**
+ * Sanitize an error object to remove sensitive information before storage.
+ * Removes file paths, connection strings, and credential patterns while
+ * preserving the error type and a useful general description for debugging.
+ *
+ * @param {Error} error - Error object to sanitize
+ * @returns {Object} Sanitized error with name, message, and optionally stack
+ */
+function sanitizeError(error) {
+  const sanitized = {
+    name: error.name || 'Error',
+    message: sanitizeMessage(error.message || ''),
+  };
+
+  if (error.stack) {
+    sanitized.stack = sanitizeMessage(error.stack);
+  }
+
+  return sanitized;
+}
+
+/**
+ * Sanitize a string message by removing sensitive patterns.
+ * @param {string} message - Raw error message
+ * @returns {string} Sanitized message
+ */
+function sanitizeMessage(message) {
+  let result = message;
+
+  // Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
+  result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
+
+  // Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
+  result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
+
+  // Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
+  result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
+
+  // Remove Windows-style absolute paths (C:\..., D:\...)
+  result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
+
+  return result;
+}
+
+// ============================================================
+// Scraper Run History
+// ============================================================
+
+/**
+ * Record scraper run to history collection.
+ * Called in the finally block of runScrape() to persist run metadata.
+ * This function must NOT throw errors - history recording should never break the scraper.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} runData - Run data to record (jobId, trigger, status, duration, etc.)
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object|null>} Insert result, or null on failure
+ */
+async function recordScraperRun(db, runData, logger) {
+  try {
+    const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
+    const result = await collection.insertOne({
+      ...runData,
+      recordedAt: new Date()
+    });
+
+    return result;
+
+  } catch (error) {
+    // Log but don't throw - recording history should not break scraper
+    logger.error('Failed to record scraper run', { errorMessage: error.message });
+    return null;
+  }
+}
+
+// ============================================================
+// Main Orchestration Function
+// ============================================================
+
+/**
+ * Execute a complete scrape operation.
+ * Orchestrates the full workflow: fetch -> parse -> convert -> DB ops.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Object} options - Scrape options
+ * @param {string} [options.trigger='manual'] - Trigger type ('scheduled' | 'manual')
+ * @param {string} [options.jobId] - Optional job ID (generated if not provided)
+ * @param {boolean} [options.dryRun=false] - Skip database writes for safe testing
+ * @param {string} [options.htmlContent] - Use provided HTML instead of fetching
+ * @returns {Promise<Object>} Scrape result with status and metrics
+ */
+async function runScrape(db, options = {}) {
+  const jobId = options.jobId || crypto.randomUUID();
+  const trigger = options.trigger || 'manual';
+  const dryRun = options.dryRun || false;
+  const htmlContent = options.htmlContent || null;
+  const logger = createLogger(jobId);
+  const startTime = Date.now();
+
+  let result = {
+    jobId,
+    trigger,
+    dryRun,
+    status: 'running',
+    startedAt: new Date().toISOString(),
+    completedAt: null,
+    duration: null,
+    unitsProcessed: 0,
+    pricesInserted: 0,
+    newUnitsCount: 0,
+    rentedUnitsCount: 0,
+    staleUnitsCount: 0,
+    errors: []
+  };
+
+  try {
+    logger.info('Scrape started', { trigger, dryRun, usingProvidedHtml: !!htmlContent });
+
+    // Step 1: Fetch HTML (or use provided content for testing)
+    const html = htmlContent || await fetchPage(config.TARGET_URL, logger);
+
+    // Step 2: Parse units
+    const rawUnits = parseUnits(html, logger);
+
+    if (rawUnits.length === 0) {
+      logger.warn('No units found in HTML - possible structure change');
+      result.errors.push('No units found in HTML');
+    }
+
+    // Step 3: Convert data types
+    const units = rawUnits.map(unit => convertDataTypes(unit));
+    result.unitsProcessed = units.length;
+
+    // Step 4: Database operations
+    const today = getTodayUTC();
+
+    // Get yesterday's unit codes for comparison
+    const yesterdayUnits = await getYesterdayUnitCodes(db, today);
+
+    // Determine new and rented units
+    const currentUnitCodes = new Set(units.map(u => u.unit_code));
+    const newUnits = units.filter(u => !yesterdayUnits.has(u.unit_code));
+    const rentedUnits = [...yesterdayUnits].filter(code => !currentUnitCodes.has(code));
+
+    result.newUnitsCount = newUnits.length;
+    result.rentedUnitsCount = rentedUnits.length;
+
+    // Database operations (skip if dryRun)
+    if (dryRun) {
+      logger.info('Dry run mode - skipping database writes', {
+        wouldUpsert: units.length,
+        wouldInsertPrices: units.filter(u => u.price !== null).length
+      });
+      result.pricesInserted = 0;
+      result.staleUnitsCount = 0;
+    } else {
+      // Upsert units
+      await upsertUnits(db, units, logger);
+
+      // Insert prices
+      const pricesResult = await insertPrices(db, units, today, logger);
+      result.pricesInserted = pricesResult.insertedCount;
+
+      // Mark stale units
+      const staleResult = await markStaleUnits(db, currentUnitCodes, today, logger);
+      result.staleUnitsCount = staleResult.modifiedCount;
+
+      // Update daily summary
+      await updateDailySummary(db, {
+        date: today,
+        newUnits: newUnits.map(u => u.unit_code),
+        rentedUnits,
+        staleUnitsCount: result.staleUnitsCount,
+        totalAvailable: units.length
+      }, logger);
+    }
+
+    result.status = 'success';
+
+  } catch (error) {
+    const cleanError = sanitizeError(error);
+    logger.error('Scrape failed', {
+      errorType: cleanError.name,
+      errorMessage: cleanError.message
+    });
+    result.status = 'failed';
+    result.errors.push(cleanError.message);
+
+  } finally {
+    result.completedAt = new Date().toISOString();
+    result.duration = Date.now() - startTime;
+
+    // Record run to history (always runs, even on failure)
+    await recordScraperRun(db, result, logger);
+
+    logger.info('Scrape completed', {
+      status: result.status,
+      duration: result.duration,
+      unitsProcessed: result.unitsProcessed,
+      pricesInserted: result.pricesInserted
+    });
+  }
+
+  return result;
+}
+
+module.exports = {
+  runScrape,
+  fetchPage,
+  parseUnits,
+  convertDataTypes,
+  upsertUnits,
+  insertPrices,
+  markStaleUnits,
+  updateDailySummary,
+  recordScraperRun,
+  // Export helpers for testing
+  getTodayUTC,
+  getYesterdayUnitCodes,
+  getYesterday,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString,
+  isRetryableError,
+  sleep,
+  sanitizeError
+};