Merge dev: Node.js scraper migration + CI fix (#32)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

__tests__/scraper/dataConversion.test.js

@@ -0,0 +1,496 @@
+/**
+ * Tests for convertDataTypes() and its internal helper functions
+ *
+ * Covers:
+ * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
+ * - parseInteger: valid integers, null/empty/invalid
+ * - parsePositiveInteger: treats 0 as null (for area)
+ * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
+ * - parseFloat: "1.5" for half baths, null/empty/invalid
+ * - parseBoolean: "true"/"false"/"1"/"0", null/empty
+ * - trimString: trim whitespace, null for empty, null/undefined
+ * - convertDataTypes: full integration applying correct parser to each field
+ */
+
+const {
+  convertDataTypes,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString
+} = require('../../services/scraperService');
+
+// ---------------------------------------------------------------
+// 1. parsePrice
+// ---------------------------------------------------------------
+describe('parsePrice', () => {
+  it('should parse "$1,234" format to 1234', () => {
+    expect(parsePrice('$1,234')).toBe(1234);
+  });
+
+  it('should parse plain numeric string', () => {
+    expect(parsePrice('1450')).toBe(1450);
+  });
+
+  it('should parse "$2,500.00" stripping non-numeric chars', () => {
+    expect(parsePrice('$2,500.00')).toBe(2500);
+  });
+
+  it('should return null for "Call for pricing"', () => {
+    expect(parsePrice('Call for pricing')).toBeNull();
+  });
+
+  it('should return null for "Contact us"', () => {
+    expect(parsePrice('Contact us')).toBeNull();
+  });
+
+  it('should return null for "Please inquire"', () => {
+    expect(parsePrice('Please inquire')).toBeNull();
+  });
+
+  it('should return null for negative values', () => {
+    // After stripping non-numeric chars, "-1500" becomes "1500"
+    // but if someone passes a numeric -1500 directly, check that too
+    expect(parsePrice('-')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(parsePrice(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePrice('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parsePrice(undefined)).toBeNull();
+  });
+
+  it('should return null for completely non-numeric string', () => {
+    expect(parsePrice('abc')).toBeNull();
+  });
+
+  it('should handle price of 0', () => {
+    expect(parsePrice('0')).toBe(0);
+  });
+
+  it('should handle large prices', () => {
+    expect(parsePrice('$15,500')).toBe(15500);
+  });
+});
+
+// ---------------------------------------------------------------
+// 2. parseInteger
+// ---------------------------------------------------------------
+describe('parseInteger', () => {
+  it('should parse valid integer string', () => {
+    expect(parseInteger('42')).toBe(42);
+  });
+
+  it('should parse "0" to 0', () => {
+    expect(parseInteger('0')).toBe(0);
+  });
+
+  it('should parse negative integer string', () => {
+    expect(parseInteger('-5')).toBe(-5);
+  });
+
+  it('should return null for null', () => {
+    expect(parseInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseInteger('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseInteger(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseInteger('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseInteger('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseInteger('Infinity')).toBeNull();
+  });
+
+  it('should truncate float strings to integer', () => {
+    expect(parseInteger('3.7')).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------
+// 3. parsePositiveInteger
+// ---------------------------------------------------------------
+describe('parsePositiveInteger', () => {
+  it('should parse positive integer normally', () => {
+    expect(parsePositiveInteger('750')).toBe(750);
+  });
+
+  it('should treat 0 as null (for area field)', () => {
+    expect(parsePositiveInteger('0')).toBeNull();
+  });
+
+  it('should return null for null input', () => {
+    expect(parsePositiveInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePositiveInteger('')).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parsePositiveInteger('abc')).toBeNull();
+  });
+
+  it('should handle negative values (parsed as negative int, not positive)', () => {
+    expect(parsePositiveInteger('-5')).toBe(-5);
+  });
+});
+
+// ---------------------------------------------------------------
+// 4. parseIntegerOrString
+// ---------------------------------------------------------------
+describe('parseIntegerOrString', () => {
+  it('should return integer when value is cleanly parseable', () => {
+    expect(parseIntegerOrString('5001')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric value', () => {
+    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
+  });
+
+  it('should return trimmed string when value has non-numeric suffix', () => {
+    expect(parseIntegerOrString('123abc')).toBe('123abc');
+  });
+
+  it('should return null for null', () => {
+    expect(parseIntegerOrString(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseIntegerOrString('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseIntegerOrString(undefined)).toBeNull();
+  });
+
+  it('should trim whitespace before checking', () => {
+    expect(parseIntegerOrString('  5001  ')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric with whitespace', () => {
+    expect(parseIntegerOrString('  abc  ')).toBe('abc');
+  });
+});
+
+// ---------------------------------------------------------------
+// 5. parseFloatValue
+// ---------------------------------------------------------------
+describe('parseFloatValue', () => {
+  it('should parse "1.5" for half baths', () => {
+    expect(parseFloatValue('1.5')).toBe(1.5);
+  });
+
+  it('should parse "2.0" to 2', () => {
+    expect(parseFloatValue('2.0')).toBe(2.0);
+  });
+
+  it('should parse integer string as float', () => {
+    expect(parseFloatValue('3')).toBe(3);
+  });
+
+  it('should return null for null', () => {
+    expect(parseFloatValue(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseFloatValue('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseFloatValue(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseFloatValue('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseFloatValue('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseFloatValue('Infinity')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 6. parseBoolean
+// ---------------------------------------------------------------
+describe('parseBoolean', () => {
+  it('should parse "true" to true', () => {
+    expect(parseBoolean('true')).toBe(true);
+  });
+
+  it('should parse "false" to false', () => {
+    expect(parseBoolean('false')).toBe(false);
+  });
+
+  it('should parse "1" to true', () => {
+    expect(parseBoolean('1')).toBe(true);
+  });
+
+  it('should parse "0" to false', () => {
+    expect(parseBoolean('0')).toBe(false);
+  });
+
+  it('should parse "TRUE" (case insensitive) to true', () => {
+    expect(parseBoolean('TRUE')).toBe(true);
+  });
+
+  it('should parse "False" (case insensitive) to false', () => {
+    expect(parseBoolean('False')).toBe(false);
+  });
+
+  it('should pass through boolean true', () => {
+    expect(parseBoolean(true)).toBe(true);
+  });
+
+  it('should pass through boolean false', () => {
+    expect(parseBoolean(false)).toBe(false);
+  });
+
+  it('should return null for null', () => {
+    expect(parseBoolean(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseBoolean('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseBoolean(undefined)).toBeNull();
+  });
+
+  it('should return null for unrecognized string', () => {
+    expect(parseBoolean('yes')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 7. trimString
+// ---------------------------------------------------------------
+describe('trimString', () => {
+  it('should trim leading and trailing whitespace', () => {
+    expect(trimString('  hello  ')).toBe('hello');
+  });
+
+  it('should return the string as-is if no whitespace', () => {
+    expect(trimString('hello')).toBe('hello');
+  });
+
+  it('should return null for empty string', () => {
+    expect(trimString('')).toBeNull();
+  });
+
+  it('should return null for whitespace-only string', () => {
+    expect(trimString('   ')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(trimString(null)).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(trimString(undefined)).toBeNull();
+  });
+
+  it('should convert non-string to string then trim', () => {
+    expect(trimString(123)).toBe('123');
+  });
+});
+
+// ---------------------------------------------------------------
+// 8. convertDataTypes - Full integration test
+// ---------------------------------------------------------------
+describe('convertDataTypes', () => {
+  it('should apply correct parser to each field', () => {
+    const rawUnit = {
+      id: '1001',
+      unit_code: 'CCT-101',
+      unit_id: '5001',
+      floor: '1',
+      area: '750',
+      bed_count: '1',
+      bath_count: '1.5',
+      price: '1450',
+      available: 'true',
+      unavailable: 'false',
+      soonest: '2026-03-01',
+      date_available: '20260301',
+      plan_id: '201',
+      plan_name: 'Alpine',
+      obj_type: 'unit',
+      community: 'Country Club Towers',
+      asset: '100',
+      href: '/units/CCT-101',
+      inventory_href: '/inventory/CCT-101',
+      image_url: 'https://example.com/images/unit-101.jpg',
+      specials_content: 'First month free!'
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    // Integer fields
+    expect(converted.id).toBe(1001);
+    expect(converted.floor).toBe(1);
+    expect(converted.bed_count).toBe(1);
+    expect(converted.plan_id).toBe(201);
+    expect(converted.asset).toBe(100);
+    expect(converted.date_available).toBe(20260301);
+
+    // parseIntegerOrString
+    expect(converted.unit_id).toBe(5001);
+
+    // parsePositiveInteger (area)
+    expect(converted.area).toBe(750);
+
+    // Float field (bath_count)
+    expect(converted.bath_count).toBe(1.5);
+
+    // Price field
+    expect(converted.price).toBe(1450);
+
+    // Boolean fields
+    expect(converted.available).toBe(true);
+    expect(converted.unavailable).toBe(false);
+
+    // String fields (trimmed)
+    expect(converted.unit_code).toBe('CCT-101');
+    expect(converted.plan_name).toBe('Alpine');
+    expect(converted.soonest).toBe('2026-03-01');
+    expect(converted.obj_type).toBe('unit');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.href).toBe('/units/CCT-101');
+    expect(converted.inventory_href).toBe('/inventory/CCT-101');
+    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
+    expect(converted.specials_content).toBe('First month free!');
+  });
+
+  it('should preserve string fields without converting them', () => {
+    const rawUnit = {
+      unit_code: 'CCT-205',
+      plan_name: '  Birch  ',
+      soonest: 'Now',
+      obj_type: 'unit',
+      community: '  Country Club Towers  ',
+      href: '/units/CCT-205',
+      inventory_href: '/inventory/CCT-205',
+      image_url: 'https://example.com/img.jpg',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.unit_code).toBe('CCT-205');
+    expect(converted.plan_name).toBe('Birch');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle a unit with all null/undefined/empty values', () => {
+    const rawUnit = {
+      id: null,
+      unit_code: undefined,
+      unit_id: '',
+      floor: null,
+      area: '',
+      bed_count: undefined,
+      bath_count: null,
+      price: '',
+      available: null,
+      unavailable: undefined,
+      soonest: '',
+      date_available: null,
+      plan_id: undefined,
+      plan_name: '',
+      obj_type: null,
+      community: undefined,
+      asset: '',
+      href: null,
+      inventory_href: undefined,
+      image_url: '',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.id).toBeNull();
+    expect(converted.unit_code).toBeNull();
+    expect(converted.unit_id).toBeNull();
+    expect(converted.floor).toBeNull();
+    expect(converted.area).toBeNull();
+    expect(converted.bed_count).toBeNull();
+    expect(converted.bath_count).toBeNull();
+    expect(converted.price).toBeNull();
+    expect(converted.available).toBeNull();
+    expect(converted.unavailable).toBeNull();
+    expect(converted.soonest).toBeNull();
+    expect(converted.date_available).toBeNull();
+    expect(converted.plan_id).toBeNull();
+    expect(converted.plan_name).toBeNull();
+    expect(converted.obj_type).toBeNull();
+    expect(converted.community).toBeNull();
+    expect(converted.asset).toBeNull();
+    expect(converted.href).toBeNull();
+    expect(converted.inventory_href).toBeNull();
+    expect(converted.image_url).toBeNull();
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle area=0 as null (parsePositiveInteger)', () => {
+    const rawUnit = { area: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.area).toBeNull();
+  });
+
+  it('should handle bed_count=0 as valid (studio)', () => {
+    const rawUnit = { bed_count: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bed_count).toBe(0);
+  });
+
+  it('should handle "Call for pricing" in price field', () => {
+    const rawUnit = { price: 'Call for pricing' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBeNull();
+  });
+
+  it('should handle "$1,234" price format', () => {
+    const rawUnit = { price: '$1,234' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBe(1234);
+  });
+
+  it('should handle unit_id as string identifier', () => {
+    const rawUnit = { unit_id: 'UNIT-ABC' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.unit_id).toBe('UNIT-ABC');
+  });
+
+  it('should handle bath_count with half baths', () => {
+    const rawUnit = { bath_count: '1.5' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bath_count).toBe(1.5);
+  });
+});

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

__tests__/scraper/fixtures/sample-listing-call.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Call for Pricing</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9001"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="750"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="P-101"
+    data-spaces-unit-id="9001"
+    data-spaces-unit-floor="1"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="500"
+    data-spaces-sort-plan-name="Penthouse A"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9001"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9001"
+    aria-label="Unit P-101">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-101</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit floor_2760 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="9002"
+    data-spaces-soonest="Now"
+    data-spaces-sort-date="1706745600"
+    data-spaces-sort-price="Call for pricing"
+    data-spaces-sort-area="900"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="P-102"
+    data-spaces-unit-id="9002"
+    data-spaces-unit-floor="2"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="501"
+    data-spaces-sort-plan-name="Penthouse B"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=9002"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=9002"
+    aria-label="Unit P-102">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">Call for pricing</span>
+      <span class="spaces-unit__unit">P-102</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing-empty.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - No Units</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <!-- Empty - no units available -->
+</section>
+</body>
+</html>

__tests__/scraper/fixtures/sample-listing.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html>
+<head><title>Country Club Towers - Apartments</title></head>
+<body>
+<section class="spaces__tab-unit">
+  <article
+    class="spaces-unit price_3500plus floor_2755 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154842"
+    data-spaces-soonest="2025-10-11"
+    data-spaces-sort-date="1760140800"
+    data-spaces-sort-price="5230"
+    data-spaces-sort-area="1210"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W2707"
+    data-spaces-unit-id="154842"
+    data-spaces-unit-floor="2755"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="10270"
+    data-spaces-sort-plan-name="Pyramid Peak - Terrace"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154842"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154842"
+    aria-label="Unit W2707">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">5230</span>
+      <span class="spaces-unit__unit">W2707</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2738 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154451"
+    data-spaces-soonest="2026-03-08"
+    data-spaces-sort-date="1772928000"
+    data-spaces-sort-price="2767"
+    data-spaces-sort-area="806"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="E0901"
+    data-spaces-unit-id="154451"
+    data-spaces-unit-floor="2738"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8016"
+    data-spaces-sort-plan-name="Sunshine Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154451"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154451"
+    aria-label="Unit E0901">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2767</span>
+      <span class="spaces-unit__unit">E0901</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2754 spaces-community-country-club-towers 0bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154814"
+    data-spaces-soonest="2025-12-18"
+    data-spaces-sort-date="1766016000"
+    data-spaces-sort-price="2255"
+    data-spaces-sort-area="623"
+    data-spaces-sort-bed="0"
+    data-spaces-unit="W2603"
+    data-spaces-unit-id="154814"
+    data-spaces-unit-floor="2754"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8018"
+    data-spaces-sort-plan-name="Little Bear Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154814"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154814"
+    aria-label="Unit W2603">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2255</span>
+      <span class="spaces-unit__unit">W2603</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2737 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154428"
+    data-spaces-soonest="2026-02-22"
+    data-spaces-sort-date="1771718400"
+    data-spaces-sort-price="4250"
+    data-spaces-sort-area="1152"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="W0802"
+    data-spaces-unit-id="154428"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8017"
+    data-spaces-sort-plan-name="Mt. Princeton"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154428"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154428"
+    aria-label="Unit W0802">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4250</span>
+      <span class="spaces-unit__unit">W0802</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2752 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154778"
+    data-spaces-soonest="2025-12-31"
+    data-spaces-sort-date="1767139200"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="764"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2405"
+    data-spaces-unit-id="154778"
+    data-spaces-unit-floor="2752"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8020"
+    data-spaces-sort-plan-name="Grays Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154778"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154778"
+    aria-label="Unit W2405">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W2405</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2737 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154442"
+    data-spaces-soonest="2025-02-01"
+    data-spaces-sort-date="1738368000"
+    data-spaces-sort-price="2495"
+    data-spaces-sort-area="802"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W0809"
+    data-spaces-unit-id="154442"
+    data-spaces-unit-floor="2737"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8022"
+    data-spaces-sort-plan-name="Pikes Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154442"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154442"
+    aria-label="Unit W0809">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2495</span>
+      <span class="spaces-unit__unit">W0809</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2000-2500 floor_2751 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154764"
+    data-spaces-soonest="2026-02-17"
+    data-spaces-sort-date="1771286400"
+    data-spaces-sort-price="2435"
+    data-spaces-sort-area="715"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W2308"
+    data-spaces-unit-id="154764"
+    data-spaces-unit-floor="2751"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8023"
+    data-spaces-sort-plan-name="Longs Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154764"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154764"
+    aria-label="Unit W2308">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2435</span>
+      <span class="spaces-unit__unit">W2308</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_2500-3000 floor_2744 spaces-community-country-club-towers 1bed 1bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154612"
+    data-spaces-soonest="2026-03-24"
+    data-spaces-sort-date="1774310400"
+    data-spaces-sort-price="2683"
+    data-spaces-sort-area="797"
+    data-spaces-sort-bed="1"
+    data-spaces-unit="W1610"
+    data-spaces-unit-id="154612"
+    data-spaces-unit-floor="2744"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8024"
+    data-spaces-sort-plan-name="Torreys Peak"
+    data-spaces-sort-bath="1"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154612"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154612"
+    aria-label="Unit W1610">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">2683</span>
+      <span class="spaces-unit__unit">W1610</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2739 spaces-community-country-club-towers 2bed 2bath spaces-market-country-club-towers-gardens"
+    data-spaces-id="154495"
+    data-spaces-soonest="2025-11-07"
+    data-spaces-sort-date="1762473600"
+    data-spaces-sort-price="4470"
+    data-spaces-sort-area="1230"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E1011"
+    data-spaces-unit-id="154495"
+    data-spaces-unit-floor="2739"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8025"
+    data-spaces-sort-plan-name="Maroon Peak"
+    data-spaces-sort-bath="2"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154495"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154495"
+    aria-label="Unit E1011">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">4470</span>
+      <span class="spaces-unit__unit">E1011</span>
+    </div>
+  </article>
+  <article
+    class="spaces-unit price_3500plus floor_2760 spaces-community-country-club-towers 2bed 2.5bath has_tour spaces-market-country-club-towers-gardens penthouse"
+    data-spaces-id="154919"
+    data-spaces-soonest="2025-10-27"
+    data-spaces-sort-date="1761523200"
+    data-spaces-sort-price="8581"
+    data-spaces-sort-area="1532"
+    data-spaces-sort-bed="2"
+    data-spaces-unit="E3205"
+    data-spaces-unit-id="154919"
+    data-spaces-unit-floor="2760"
+    data-spaces-obj="unit"
+    data-spaces-unavailable="false"
+    data-spaces-available="true"
+    data-spaces-community="Country Club Towers"
+    data-spaces-asset="420"
+    data-spaces-specials-content=""
+    data-spaces-plan-id="8037"
+    data-spaces-sort-plan-name="Mt. Antero"
+    data-spaces-sort-bath="2.5"
+    data-spaces-href="?spaces_tab=unit-detail&detail=154919"
+    data-spaces-inventory-href="?spaces_tab=unit-detail&detail=154919"
+    aria-label="Unit E3205">
+    <div class="spaces-unit__inner">
+      <span class="spaces-unit__price">8581</span>
+      <span class="spaces-unit__unit">E3205</span>
+    </div>
+  </article>
+</section>
+</body>
+</html>

__tests__/scraper/insertPrices.test.js

@@ -0,0 +1,582 @@
+/**
+ * Tests for insertPrices() bulk operation
+ *
+ * Covers:
+ * - Only units with non-null prices are inserted
+ * - bulkWrite uses updateOne with upsert for idempotency
+ * - Filter uses unit_code + date_checked composite key
+ * - Price record includes unit_code, date_checked, price, last_updated, data_source
+ * - Empty price records logs warning and returns early
+ * - Error handling when bulkWrite fails
+ * - Logging of inserted and updated counts
+ * - Re-running on same day updates existing records (idempotent)
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require insertPrices after implementation
+let insertPrices;
+
+let mongoServer;
+let client;
+let db;
+
+const PRICES_COLLECTION = 'unit_prices_scraper';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ insertPrices } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the prices collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('insertPrices', () => {
+  const TODAY = '2026-02-05';
+
+  // ---------------------------------------------------------------
+  // 1. Only units with non-null prices are inserted
+  // ---------------------------------------------------------------
+  describe('filtering units with null prices', () => {
+    it('should only insert records for units with non-null prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200 },
+        { unit_code: 'A102', price: null },
+        { unit_code: 'A103', price: 1500 },
+        { unit_code: 'A104', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'A103']);
+    });
+
+    it('should skip units with price of null and not create any record for them', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'B101', price: null },
+        { unit_code: 'B102', price: null }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(0);
+    });
+
+    it('should handle a mix of priced and null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'C101', price: 950 },
+        { unit_code: 'C102', price: null },
+        { unit_code: 'C103', price: 1100 },
+        { unit_code: 'C104', price: null },
+        { unit_code: 'C105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. bulkWrite uses updateOne with upsert for idempotency
+  // ---------------------------------------------------------------
+  describe('bulkWrite with upsert operations', () => {
+    it('should create updateOne operations for each priced unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'D101', price: 1200 },
+        { unit_code: 'D102', price: 1500 }
+      ];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.insertedCount).toBeGreaterThan(0);
+
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+    });
+
+    it('should use upsert so re-running does not create duplicates', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'D201', price: 1200 }];
+
+      // First run
+      await insertPrices(db, units, TODAY, logger);
+
+      // Second run on same day
+      await insertPrices(db, units, TODAY, logger);
+
+      // Should still have only 1 document (not 2)
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Filter uses unit_code + date_checked composite key
+  // ---------------------------------------------------------------
+  describe('composite key: unit_code + date_checked', () => {
+    it('should use unit_code and date_checked as the filter for upsert', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E101', price: 1200 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'E101',
+        date_checked: TODAY
+      });
+
+      expect(doc).not.toBeNull();
+      expect(doc.unit_code).toBe('E101');
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should create separate records for the same unit on different days', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E201', price: 1200 }];
+
+      await insertPrices(db, units, '2026-02-04', logger);
+      await insertPrices(db, units, '2026-02-05', logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ unit_code: 'E201' })
+        .sort({ date_checked: 1 })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+      expect(docs[0].date_checked).toBe('2026-02-04');
+      expect(docs[1].date_checked).toBe('2026-02-05');
+    });
+
+    it('should create separate records for different units on the same day', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'E301', price: 1000 },
+        { unit_code: 'E302', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const docs = await db.collection(PRICES_COLLECTION)
+        .find({ date_checked: TODAY })
+        .toArray();
+
+      expect(docs).toHaveLength(2);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Price record includes required fields
+  // ---------------------------------------------------------------
+  describe('price record fields', () => {
+    it('should include unit_code in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.unit_code).toBe('F101');
+    });
+
+    it('should include date_checked in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F201', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.date_checked).toBe(TODAY);
+    });
+
+    it('should include price as a number in the price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F301', price: 1550 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.price).toBe(1550);
+      expect(typeof doc.price).toBe('number');
+    });
+
+    it('should include last_updated as an ISO timestamp string', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'F401', price: 1600 }];
+      await insertPrices(db, units, TODAY, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.last_updated).toBeDefined();
+      expect(typeof doc.last_updated).toBe('string');
+      expect(doc.last_updated >= beforeTime).toBe(true);
+      expect(doc.last_updated <= afterTime).toBe(true);
+    });
+
+    it('should include data_source set to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F501', price: 1700 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should include all five required fields in every price record', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'F601', price: 1800 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const doc = await db.collection(PRICES_COLLECTION).findOne({});
+      expect(doc).toHaveProperty('unit_code');
+      expect(doc).toHaveProperty('date_checked');
+      expect(doc).toHaveProperty('price');
+      expect(doc).toHaveProperty('last_updated');
+      expect(doc).toHaveProperty('data_source');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty price records logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty price records', () => {
+    it('should log a warning when no units have prices', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'G101', price: null },
+        { unit_code: 'G102', price: null }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should return insertedCount of 0 for all null-priced units', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'G201', price: null }];
+
+      const result = await insertPrices(db, units, TODAY, logger);
+
+      expect(result.insertedCount).toBe(0);
+    });
+
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No price records to insert');
+    });
+
+    it('should not perform any database writes for empty/null-priced arrays', async () => {
+      const logger = createMockLogger();
+
+      await insertPrices(db, [], TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H101', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H201', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await insertPrices(mockDb, units, TODAY, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to insert prices',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'H301', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(insertPrices(mockDb, units, TODAY, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Logging of inserted and updated counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log inserted and updated counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I101', price: 1000 },
+        { unit_code: 'I102', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: expect.any(Number),
+          updated: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'I201', price: 1000 },
+        { unit_code: 'I202', price: 1100 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // For new inserts: inserted=2, updated=0
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 2,
+          updated: 0
+        })
+      );
+    });
+
+    it('should log correct counts when updating existing records', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'I301', price: 1000 }];
+
+      // First insert
+      await insertPrices(db, units, TODAY, logger);
+
+      // Reset mock
+      logger.info.mockClear();
+
+      // Second run updates existing
+      const updatedUnits = [{ unit_code: 'I301', price: 1050 }];
+      await insertPrices(db, updatedUnits, TODAY, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Prices inserted',
+        expect.objectContaining({
+          inserted: 0,
+          updated: 1
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Re-running on same day updates existing records (idempotent)
+  // ---------------------------------------------------------------
+  describe('idempotent re-runs on same day', () => {
+    it('should update the price when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run with original price
+      await insertPrices(db, [{ unit_code: 'J101', price: 1200 }], TODAY, logger);
+
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc1.price).toBe(1200);
+
+      // Re-run with updated price (same day)
+      await insertPrices(db, [{ unit_code: 'J101', price: 1250 }], TODAY, logger);
+
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(doc2.price).toBe(1250);
+
+      // Should still have only 1 record
+      const count = await db.collection(PRICES_COLLECTION).countDocuments({
+        unit_code: 'J101',
+        date_checked: TODAY
+      });
+      expect(count).toBe(1);
+    });
+
+    it('should update last_updated when re-running on the same day', async () => {
+      const logger = createMockLogger();
+
+      // First run
+      await insertPrices(db, [{ unit_code: 'J201', price: 1300 }], TODAY, logger);
+      const doc1 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const firstUpdated = doc1.last_updated;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Re-run on same day
+      await insertPrices(db, [{ unit_code: 'J201', price: 1350 }], TODAY, logger);
+      const doc2 = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J201' });
+      const secondUpdated = doc2.last_updated;
+
+      expect(secondUpdated > firstUpdated).toBe(true);
+    });
+
+    it('should return combined insert/update count reflecting the operation', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'J301', price: 1000 },
+        { unit_code: 'J302', price: 1100 }
+      ];
+
+      // First run - inserts
+      const result1 = await insertPrices(db, units, TODAY, logger);
+      expect(result1.insertedCount).toBe(2);
+
+      // Re-run - updates
+      const result2 = await insertPrices(db, units, TODAY, logger);
+      expect(result2.insertedCount).toBe(2); // combined upserted + modified
+
+      // Still only 2 total records
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(2);
+    });
+
+    it('should handle a mix of new and existing records on re-run', async () => {
+      const logger = createMockLogger();
+
+      // First run with unit A
+      await insertPrices(db, [{ unit_code: 'J401', price: 1000 }], TODAY, logger);
+
+      // Re-run with unit A (existing) and unit B (new)
+      const mixedUnits = [
+        { unit_code: 'J401', price: 1050 },
+        { unit_code: 'J402', price: 1200 }
+      ];
+      await insertPrices(db, mixedUnits, TODAY, logger);
+
+      // Should have 2 total records
+      const docs = await db.collection(PRICES_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      // Verify updated price
+      const unitA = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'J401' });
+      expect(unitA.price).toBe(1050);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Uses ordered: false for performance
+  // ---------------------------------------------------------------
+  describe('ordered: false for performance', () => {
+    it('should successfully process all priced units in a batch', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'K101', price: 900 },
+        { unit_code: 'K102', price: 1000 },
+        { unit_code: 'K103', price: 1100 },
+        { unit_code: 'K104', price: 1200 },
+        { unit_code: 'K105', price: 1300 }
+      ];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      const count = await db.collection(PRICES_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Uses correct collection
+  // ---------------------------------------------------------------
+  describe('collection usage', () => {
+    it('should write to the configured PRICES collection', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'L101', price: 1400 }];
+
+      await insertPrices(db, units, TODAY, logger);
+
+      // Verify data is in the correct collection
+      const doc = await db.collection(PRICES_COLLECTION).findOne({ unit_code: 'L101' });
+      expect(doc).not.toBeNull();
+    });
+  });
+});

__tests__/scraper/markStaleUnits.test.js

@@ -0,0 +1,449 @@
+/**
+ * Tests for markStaleUnits() availability tracking
+ *
+ * Covers:
+ * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
+ * - $set updates available to false and sets marked_stale_date
+ * - Units in currentCodes are NOT marked stale
+ * - Already unavailable units are not modified
+ * - Error handling when updateMany fails
+ * - Logging of modified count
+ * - Empty currentUnitCodes handles gracefully
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require markStaleUnits after implementation
+let markStaleUnits;
+
+let mongoServer;
+let client;
+let db;
+
+const UNITS_COLLECTION = 'units_scraper';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ markStaleUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('markStaleUnits', () => {
+  // ---------------------------------------------------------------
+  // 1. updateMany is called with correct filter
+  //    (unit_code $nin currentCodes AND available: true)
+  // ---------------------------------------------------------------
+  describe('filter criteria', () => {
+    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
+      const logger = createMockLogger();
+
+      // Seed units in the database
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'A101', available: true, price: 1200 },
+        { unit_code: 'A102', available: true, price: 1300 },
+        { unit_code: 'B201', available: true, price: 1500 }
+      ]);
+
+      // Current scrape found A101 and A102, but NOT B201
+      const currentCodes = new Set(['A101', 'A102']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // B201 should be marked stale (it was available but not in current scrape)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify B201 was updated
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
+      expect(staleUnit.available).toBe(false);
+      expect(staleUnit.marked_stale_date).toBe(date);
+    });
+
+    it('should only affect units NOT in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'X1', available: true },
+        { unit_code: 'X2', available: true },
+        { unit_code: 'X3', available: true },
+        { unit_code: 'X4', available: true }
+      ]);
+
+      // Current scrape found X1 and X3
+      const currentCodes = new Set(['X1', 'X3']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // X1 and X3 should still be available
+      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
+      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
+      expect(x1.available).toBe(true);
+      expect(x3.available).toBe(true);
+
+      // X2 and X4 should be marked stale
+      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
+      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
+      expect(x2.available).toBe(false);
+      expect(x4.available).toBe(false);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. $set updates available to false and sets marked_stale_date
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set available to false for stale units', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'S100', available: true, price: 1000 },
+        { unit_code: 'S200', available: true, price: 1100 }
+      ]);
+
+      const currentCodes = new Set(['S100']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
+      expect(staleUnit.available).toBe(false);
+    });
+
+    it('should set marked_stale_date to the provided date', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'D100', available: true }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
+      expect(doc.marked_stale_date).toBe('2026-02-05');
+    });
+
+    it('should update both available and marked_stale_date in the same operation', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'C100', available: true, price: 900 }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-01-15';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
+      expect(doc.available).toBe(false);
+      expect(doc.marked_stale_date).toBe('2026-01-15');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Units in currentCodes are NOT marked stale
+  // ---------------------------------------------------------------
+  describe('units in currentCodes not affected', () => {
+    it('should not modify units that are in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'KEEP-1', available: true, price: 1200 },
+        { unit_code: 'KEEP-2', available: true, price: 1300 },
+        { unit_code: 'STALE-1', available: true, price: 1400 }
+      ]);
+
+      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // Kept units should retain their original state
+      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
+      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
+
+      expect(keep1.available).toBe(true);
+      expect(keep1.marked_stale_date).toBeUndefined();
+      expect(keep2.available).toBe(true);
+      expect(keep2.marked_stale_date).toBeUndefined();
+    });
+
+    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ALL-1', available: true },
+        { unit_code: 'ALL-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['ALL-1', 'ALL-2']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Already unavailable units are not modified
+  // ---------------------------------------------------------------
+  describe('already unavailable units', () => {
+    it('should not modify units that are already unavailable (available: false)', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
+        { unit_code: 'UNAVAIL-2', available: false },
+        { unit_code: 'AVAIL-1', available: true }
+      ]);
+
+      // None of these units are in the current scrape
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify the already-unavailable unit's date was NOT changed
+      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
+      expect(unavail1.marked_stale_date).toBe('2026-01-01');
+    });
+
+    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'U1', available: false },
+        { unit_code: 'U2', available: false }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Error handling when updateMany fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      try {
+        await markStaleUnits(mockDb, currentCodes, date, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to mark stale units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging of modified count
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of stale units marked', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'LOG-1', available: true },
+        { unit_code: 'LOG-2', available: true },
+        { unit_code: 'LOG-3', available: true }
+      ]);
+
+      const currentCodes = new Set(['LOG-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 2 })
+      );
+    });
+
+    it('should log count of 0 when no units are marked stale', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ZERO-1', available: true }
+      ]);
+
+      const currentCodes = new Set(['ZERO-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Empty currentUnitCodes handles gracefully
+  // ---------------------------------------------------------------
+  describe('empty currentUnitCodes', () => {
+    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'EMPTY-1', available: true },
+        { unit_code: 'EMPTY-2', available: true },
+        { unit_code: 'EMPTY-3', available: true }
+      ]);
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(3);
+
+      // All should be marked unavailable
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      for (const doc of docs) {
+        expect(doc.available).toBe(false);
+        expect(doc.marked_stale_date).toBe('2026-02-05');
+      }
+    });
+
+    it('should not throw when currentUnitCodes is an empty Set', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      // Should not throw even with empty collection and empty set
+      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
+    });
+
+    it('should handle empty database with empty currentUnitCodes', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the MongoDB updateMany result', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'RET-1', available: true },
+        { unit_code: 'RET-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['RET-1']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // updateMany result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('acknowledged');
+      expect(result.acknowledged).toBe(true);
+    });
+  });
+});

__tests__/scraper/parseUnits.test.js

@@ -0,0 +1,517 @@
+/**
+ * Tests for parseUnits() - HTML parsing with cheerio
+ *
+ * Covers:
+ * - Extraction of all data-spaces-* attributes
+ * - Image URL extraction from nested img elements
+ * - Container detection (section.spaces__tab-unit)
+ * - Empty / missing container handling
+ * - Deduplication by unit_code
+ * - Logging behavior (info, warn, error)
+ * - Missing attribute handling
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseUnits } = require('../../services/scraperService');
+
+const fixturesDir = path.join(__dirname, 'fixtures');
+
+// Helper: create a mock logger
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+describe('parseUnits', () => {
+  let sampleHtml;
+  let mockLogger;
+
+  beforeAll(() => {
+    sampleHtml = fs.readFileSync(
+      path.join(fixturesDir, 'sample-listing.html'),
+      'utf-8'
+    );
+  });
+
+  beforeEach(() => {
+    mockLogger = createMockLogger();
+  });
+
+  // ---------------------------------------------------------------
+  // 1. Container detection
+  // ---------------------------------------------------------------
+  describe('container detection', () => {
+    it('should find section.spaces__tab-unit container in valid HTML', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+
+      // Should not log an error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+      expect(units.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array when section container is missing', () => {
+      const html = '<html><body><div>No units here</div></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+
+    it('should return empty array for empty HTML string', () => {
+      const units = parseUnits('', mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    it('should return empty array when container exists but has no articles', () => {
+      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container found, no error about container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Extraction of all data-spaces-* attributes
+  // ---------------------------------------------------------------
+  describe('attribute extraction', () => {
+    it('should extract all expected attributes from a fully populated article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'W2707');
+
+      expect(unit1).toBeDefined();
+
+      // Core identifiers
+      expect(unit1.id).toBe('154842');
+      expect(unit1.unit_code).toBe('W2707');
+      expect(unit1.unit_id).toBe('154842');
+
+      // Physical attributes
+      expect(unit1.floor).toBe('2755');
+      expect(unit1.area).toBe('1210');
+      expect(unit1.bed_count).toBe('2');
+      expect(unit1.bath_count).toBe('2');
+
+      // Pricing
+      expect(unit1.price).toBe('5230');
+
+      // Availability
+      expect(unit1.available).toBe('true');
+      expect(unit1.unavailable).toBe('false');
+      expect(unit1.soonest).toBe('2025-10-11');
+      expect(unit1.date_available).toBe('1760140800');
+
+      // Plan information
+      expect(unit1.plan_id).toBe('10270');
+      expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
+
+      // Property information
+      expect(unit1.obj_type).toBe('unit');
+      expect(unit1.community).toBe('Country Club Towers');
+      expect(unit1.asset).toBe('420');
+
+      // URLs
+      expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
+      expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
+
+      // Specials
+      expect(unit1.specials_content).toBe('');
+    });
+
+    it('should extract correct count of units from the fixture', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      expect(units).toHaveLength(10);
+    });
+
+    it('should return raw string values without type conversion', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'W2707');
+
+      // All values should be strings (raw extraction, no conversion)
+      expect(typeof unit.id).toBe('string');
+      expect(typeof unit.price).toBe('string');
+      expect(typeof unit.floor).toBe('string');
+      expect(typeof unit.available).toBe('string');
+      expect(typeof unit.bed_count).toBe('string');
+      expect(typeof unit.bath_count).toBe('string');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Image URL extraction
+  // ---------------------------------------------------------------
+  describe('image URL extraction', () => {
+    it('should extract image_url from img src attribute', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
+            <img src="https://example.com/images/unit-1.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
+    });
+
+    it('should fall back to data-src when src is not present', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
+            <img data-src="https://example.com/images/unit-2.jpg" />
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
+    });
+
+    it('should not set image_url when no img element exists', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
+            <div>No image here</div>
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+
+      expect(units[0].image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Missing attributes
+  // ---------------------------------------------------------------
+  describe('missing attributes', () => {
+    it('should return undefined for attributes not present on the article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit).toBeDefined();
+      expect(unit.floor).toBeUndefined();
+      expect(unit.area).toBeUndefined();
+      expect(unit.bed_count).toBeUndefined();
+      expect(unit.bath_count).toBeUndefined();
+      expect(unit.soonest).toBeUndefined();
+      expect(unit.date_available).toBeUndefined();
+      expect(unit.plan_id).toBeUndefined();
+      expect(unit.plan_name).toBeUndefined();
+      expect(unit.href).toBeUndefined();
+      expect(unit.inventory_href).toBeUndefined();
+      expect(unit.specials_content).toBeUndefined();
+    });
+
+    it('should still extract the attributes that are present on a sparse article', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="1005"
+            data-spaces-unit="SPARSE-1"
+            data-spaces-unit-id="5005"
+            data-spaces-sort-price="1300"
+            data-spaces-available="true"
+            data-spaces-obj="unit"
+            data-spaces-community="Country Club Towers"
+            data-spaces-asset="100">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.id).toBe('1005');
+      expect(unit.unit_code).toBe('SPARSE-1');
+      expect(unit.unit_id).toBe('5005');
+      expect(unit.price).toBe('1300');
+      expect(unit.available).toBe('true');
+      expect(unit.obj_type).toBe('unit');
+      expect(unit.community).toBe('Country Club Towers');
+      expect(unit.asset).toBe('100');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Deduplication
+  // ---------------------------------------------------------------
+  describe('deduplication', () => {
+    it('should deduplicate units with the same unit_code', () => {
+      // Build HTML with duplicate articles
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+      const codes = units.map(u => u.unit_code);
+
+      // Should have 2 unique unit codes
+      expect(units).toHaveLength(2);
+      expect(new Set(codes).size).toBe(codes.length);
+    });
+
+    it('should keep the first occurrence when deduplicating', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // First occurrence (id=1, price=1000) should be kept
+      expect(units[0].id).toBe('1');
+      expect(units[0].price).toBe('1000');
+    });
+
+    it('should log a warning when duplicates are removed', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Duplicate units removed',
+        expect.objectContaining({
+          original: 3,
+          deduplicated: 2
+        })
+      );
+    });
+
+    it('should not log a warning when there are no duplicates', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      // The fixture has no duplicates, so no warn about duplicates
+      const warnCalls = mockLogger.warn.mock.calls;
+      const duplicateWarns = warnCalls.filter(
+        call => call[0] === 'Duplicate units removed'
+      );
+      expect(duplicateWarns).toHaveLength(0);
+    });
+
+    it('should filter out articles with no unit_code during deduplication', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // Article without unit_code should be filtered out
+      expect(units).toHaveLength(1);
+      expect(units[0].unit_code).toBe('VALID-1');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging behavior
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of parsed units', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 10 })
+      );
+    });
+
+    it('should log error when container is not found', () => {
+      const html = '<html><body><p>Nothing here</p></body></html>';
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.error).toHaveBeenCalledTimes(1);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
+  // ---------------------------------------------------------------
+  describe('fixture: sample-listing-empty.html', () => {
+    let emptyHtml;
+
+    beforeAll(() => {
+      emptyHtml = fs.readFileSync(
+        path.join(fixturesDir, 'sample-listing-empty.html'),
+        'utf-8'
+      );
+    });
+
+    it('should return empty array when fixture has container but no articles', () => {
+      const units = parseUnits(emptyHtml, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container exists so no error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+
+    it('should log zero parsed units for empty fixture', () => {
+      parseUnits(emptyHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  describe('fixture: sample-listing-call.html', () => {
+    let callHtml;
+
+    beforeAll(() => {
+      callHtml = fs.readFileSync(
+        path.join(fixturesDir, 'sample-listing-call.html'),
+        'utf-8'
+      );
+    });
+
+    it('should extract both units from the call-for-pricing fixture', () => {
+      const units = parseUnits(callHtml, mockLogger);
+
+      expect(units).toHaveLength(2);
+      expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
+    });
+
+    it('should preserve "Call for pricing" as raw string in the price field', () => {
+      const units = parseUnits(callHtml, mockLogger);
+
+      expect(units[0].price).toBe('Call for pricing');
+      expect(units[1].price).toBe('Call for pricing');
+    });
+
+    it('should extract all attributes correctly from call-for-pricing units', () => {
+      const units = parseUnits(callHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'P-101');
+
+      expect(unit.id).toBe('9001');
+      expect(unit.unit_id).toBe('9001');
+      expect(unit.floor).toBe('1');
+      expect(unit.area).toBe('750');
+      expect(unit.bed_count).toBe('1');
+      expect(unit.bath_count).toBe('1');
+      expect(unit.available).toBe('true');
+      expect(unit.unavailable).toBe('false');
+      expect(unit.soonest).toBe('Now');
+      expect(unit.date_available).toBe('1706745600');
+      expect(unit.plan_id).toBe('500');
+      expect(unit.plan_name).toBe('Penthouse A');
+      expect(unit.community).toBe('Country Club Towers');
+      expect(unit.asset).toBe('420');
+      expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
+      expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
+      expect(unit.specials_content).toBe('');
+    });
+
+    it('should not have image_url when fixture units lack img elements', () => {
+      const units = parseUnits(callHtml, mockLogger);
+
+      expect(units[0].image_url).toBeUndefined();
+      expect(units[1].image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Multiple units with varied data
+  // ---------------------------------------------------------------
+  describe('varied unit data', () => {
+    it('should handle unavailable units correctly', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article
+            data-spaces-id="4001"
+            data-spaces-unit="UNAVAIL-1"
+            data-spaces-sort-price="2000"
+            data-spaces-available="false"
+            data-spaces-unavailable="true"
+            data-spaces-soonest=""
+            data-spaces-sort-date="">
+          </article>
+        </section>
+        </body></html>
+      `;
+      const units = parseUnits(html, mockLogger);
+      const unit = units[0];
+
+      expect(unit.available).toBe('false');
+      expect(unit.unavailable).toBe('true');
+      expect(unit.soonest).toBe('');
+      expect(unit.date_available).toBe('');
+    });
+
+    it('should handle half bath counts', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // E3205 is the penthouse with 2.5 baths
+      const unit = units.find(u => u.unit_code === 'E3205');
+
+      expect(unit.bath_count).toBe('2.5');
+    });
+
+    it('should handle studio units (bed_count = 0)', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // W2603 is the studio with bed_count 0
+      const unit = units.find(u => u.unit_code === 'W2603');
+
+      expect(unit.bed_count).toBe('0');
+    });
+  });
+});

__tests__/scraper/recordScraperRun.test.js

@@ -0,0 +1,307 @@
+/**
+ * Tests for recordScraperRun() history persistence
+ *
+ * Covers:
+ * - insertOne is called with runData plus recordedAt timestamp
+ * - All runData fields are preserved (jobId, trigger, status, duration, etc.)
+ * - Error is caught and logged but NOT thrown (graceful failure)
+ * - Returns null on error instead of crashing
+ * - Returns insert result on success
+ * - recordedAt is a valid Date object
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require after implementation
+let recordScraperRun;
+
+let mongoServer;
+let client;
+let db;
+let logger;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ recordScraperRun } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+
+  // Create fresh mock logger for each test
+  logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+});
+
+describe('recordScraperRun', () => {
+  const SCRAPER_RUNS_COLLECTION = 'scraper_runs';
+
+  // Sample runData matching the structure from runScrape()'s finally block
+  function createSampleRunData(overrides = {}) {
+    return {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      dryRun: false,
+      status: 'success',
+      startedAt: '2026-02-06T06:00:00.000Z',
+      completedAt: '2026-02-06T06:00:12.345Z',
+      duration: 12345,
+      unitsProcessed: 50,
+      pricesInserted: 48,
+      newUnitsCount: 3,
+      rentedUnitsCount: 1,
+      staleUnitsCount: 2,
+      errors: [],
+      ...overrides
+    };
+  }
+
+  // ---------------------------------------------------------------
+  // 1. insertOne is called with runData plus recordedAt timestamp
+  // ---------------------------------------------------------------
+  describe('insertOne with runData and recordedAt', () => {
+    it('should insert a document into the scraper_runs collection', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const docs = await db.collection(SCRAPER_RUNS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+    });
+
+    it('should include a recordedAt field in the inserted document', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. All runData fields are preserved
+  // ---------------------------------------------------------------
+  describe('preserving all runData fields', () => {
+    it('should preserve jobId, trigger, status, and duration', async () => {
+      const runData = createSampleRunData({
+        jobId: 'preserve-test-001',
+        trigger: 'scheduled',
+        status: 'failed',
+        duration: 9999
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.jobId).toBe('preserve-test-001');
+      expect(doc.trigger).toBe('scheduled');
+      expect(doc.status).toBe('failed');
+      expect(doc.duration).toBe(9999);
+    });
+
+    it('should preserve unit processing metrics', async () => {
+      const runData = createSampleRunData({
+        unitsProcessed: 42,
+        pricesInserted: 40,
+        newUnitsCount: 5,
+        rentedUnitsCount: 2,
+        staleUnitsCount: 3
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.unitsProcessed).toBe(42);
+      expect(doc.pricesInserted).toBe(40);
+      expect(doc.newUnitsCount).toBe(5);
+      expect(doc.rentedUnitsCount).toBe(2);
+      expect(doc.staleUnitsCount).toBe(3);
+    });
+
+    it('should preserve timing fields (startedAt, completedAt)', async () => {
+      const runData = createSampleRunData({
+        startedAt: '2026-02-06T10:00:00.000Z',
+        completedAt: '2026-02-06T10:00:15.500Z'
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.startedAt).toBe('2026-02-06T10:00:00.000Z');
+      expect(doc.completedAt).toBe('2026-02-06T10:00:15.500Z');
+    });
+
+    it('should preserve dryRun flag', async () => {
+      const runData = createSampleRunData({ dryRun: true });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.dryRun).toBe(true);
+    });
+
+    it('should preserve errors array when it has entries', async () => {
+      const runData = createSampleRunData({
+        status: 'failed',
+        errors: ['Connection timeout', 'Retry exhausted']
+      });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual(['Connection timeout', 'Retry exhausted']);
+    });
+
+    it('should preserve empty errors array for successful runs', async () => {
+      const runData = createSampleRunData({ errors: [] });
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.errors).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Error is caught and logged but NOT thrown (graceful failure)
+  // ---------------------------------------------------------------
+  describe('graceful error handling', () => {
+    it('should NOT throw when insertOne fails', async () => {
+      const runData = createSampleRunData();
+
+      // Create a mock db that throws on insertOne
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      // This should NOT throw
+      await expect(recordScraperRun(mockDb, runData, logger)).resolves.not.toThrow();
+    });
+
+    it('should log error message when insert fails', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Disk full'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await recordScraperRun(mockDb, runData, logger);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to record scraper run',
+        { errorMessage: 'Disk full' }
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Returns null on error instead of crashing
+  // ---------------------------------------------------------------
+  describe('return value on error', () => {
+    it('should return null when insertOne throws', async () => {
+      const runData = createSampleRunData();
+
+      const mockCollection = {
+        insertOne: jest.fn().mockRejectedValue(new Error('Connection refused'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const result = await recordScraperRun(mockDb, runData, logger);
+      expect(result).toBeNull();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Returns insert result on success
+  // ---------------------------------------------------------------
+  describe('return value on success', () => {
+    it('should return the insertOne result object', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result).toBeDefined();
+      expect(result).not.toBeNull();
+    });
+
+    it('should return result with acknowledged property', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.acknowledged).toBe(true);
+    });
+
+    it('should return result with insertedId', async () => {
+      const runData = createSampleRunData();
+
+      const result = await recordScraperRun(db, runData, logger);
+
+      expect(result.insertedId).toBeDefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. recordedAt is a valid Date object
+  // ---------------------------------------------------------------
+  describe('recordedAt field', () => {
+    it('should set recordedAt as a Date instance', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+
+    it('should set recordedAt close to current time', async () => {
+      const beforeTime = new Date();
+
+      const runData = createSampleRunData();
+      await recordScraperRun(db, runData, logger);
+
+      const afterTime = new Date();
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+      expect(doc.recordedAt.getTime()).toBeGreaterThanOrEqual(beforeTime.getTime());
+      expect(doc.recordedAt.getTime()).toBeLessThanOrEqual(afterTime.getTime());
+    });
+
+    it('should not overwrite any existing runData fields with recordedAt', async () => {
+      const runData = createSampleRunData();
+
+      await recordScraperRun(db, runData, logger);
+
+      const doc = await db.collection(SCRAPER_RUNS_COLLECTION).findOne({});
+
+      // Verify the original fields still exist alongside recordedAt
+      expect(doc.jobId).toBe(runData.jobId);
+      expect(doc.status).toBe(runData.status);
+      expect(doc.recordedAt).toBeInstanceOf(Date);
+    });
+  });
+});

__tests__/scraper/scraperConfig.test.js

@@ -0,0 +1,192 @@
+/**
+ * Tests for config/scraper.js
+ * Tests scraper configuration constants and environment variable overrides
+ */
+
+describe('config/scraper', () => {
+  // Store original env values to restore after tests
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    // Clear all scraper-related env vars before each test
+    delete process.env.SCRAPER_SCHEDULE;
+    delete process.env.SCRAPER_TIMEZONE;
+    delete process.env.SCRAPER_ENABLED;
+    delete process.env.SCRAPER_TIMEOUT;
+    delete process.env.SCRAPER_UNITS_COLLECTION;
+    delete process.env.SCRAPER_PRICES_COLLECTION;
+    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
+    delete process.env.SCRAPER_RUNS_COLLECTION;
+
+    // Clear the require cache to reload config with fresh env
+    jest.resetModules();
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    process.env = originalEnv;
+  });
+
+  describe('TARGET_URL', () => {
+    test('should be set to the correct apartment listing URL', () => {
+      const config = require('../../config/scraper');
+      expect(config.TARGET_URL).toBe(
+        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
+      );
+    });
+  });
+
+  describe('SCRAPER_SCHEDULE', () => {
+    test('should default to "0 6 * * *" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
+    });
+  });
+
+  describe('SCRAPER_TIMEZONE', () => {
+    test('should default to "UTC" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+    });
+  });
+
+  describe('SCRAPER_ENABLED', () => {
+    test('should default to true when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be false when env var is set to "false"', () => {
+      process.env.SCRAPER_ENABLED = 'false';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(false);
+    });
+
+    test('should be true when env var is set to any other value', () => {
+      process.env.SCRAPER_ENABLED = 'true';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be true when env var is empty string', () => {
+      process.env.SCRAPER_ENABLED = '';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+  });
+
+  describe('SCRAPER_TIMEOUT', () => {
+    test('should default to 30000 when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(30000);
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEOUT = '60000';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(60000);
+    });
+
+    test('should parse string to integer', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
+      expect(config.SCRAPER_TIMEOUT).toBe(45000);
+    });
+  });
+
+  describe('USER_AGENT', () => {
+    test('should be a valid User-Agent string', () => {
+      const config = require('../../config/scraper');
+      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
+    });
+  });
+
+  describe('RETRY_CONFIG', () => {
+    test('should have maxRetries property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
+      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
+    });
+
+    test('should have baseDelay property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
+      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
+    });
+
+    test('should have timeout property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
+      expect(config.RETRY_CONFIG.timeout).toBe(30000);
+    });
+
+    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG.timeout).toBe(45000);
+    });
+  });
+
+  describe('COLLECTIONS', () => {
+    describe('default values', () => {
+      test('UNITS should default to "units_scraper"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
+      });
+
+      test('PRICES should default to "unit_prices_scraper"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
+      });
+
+      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
+      });
+
+      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
+      });
+    });
+
+    describe('env var overrides', () => {
+      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
+        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
+      });
+
+      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
+        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
+      });
+
+      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
+        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
+      });
+
+      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
+        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
+      });
+    });
+  });
+});

__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

__tests__/scraper/scraperService.test.js

@@ -0,0 +1,739 @@
+/**
+ * Tests for runScrape() orchestration function
+ *
+ * Covers:
+ * - Full workflow completes with mocked dependencies
+ * - Returns correct result structure with jobId, status, metrics
+ * - Handles dryRun option (skip DB writes)
+ * - Handles htmlContent option (use provided HTML)
+ * - Records scraper run to history on success
+ * - Records scraper run to history on failure
+ * - Catches and logs errors from fetchPage
+ * - Catches and logs errors from database operations
+ * - Calculates newUnitsCount and rentedUnitsCount correctly
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+let mongoServer;
+let client;
+let db;
+
+// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
+let runScrape;
+let recordScraperRun;
+let createScraperIndexes;
+
+// Store original module references so we can mock individual functions
+let scraperService;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  scraperService = require('../../services/scraperService');
+  ({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean all relevant collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+  // Reset all mocks
+  jest.restoreAllMocks();
+});
+
+// ============================================================
+// Helper: Create sample HTML with units
+// ============================================================
+function createSampleHtml(unitCodes) {
+  const articles = unitCodes.map(code => `
+    <article
+      data-spaces-id="100"
+      data-spaces-unit="${code}"
+      data-spaces-unit-id="200"
+      data-spaces-unit-floor="5"
+      data-spaces-sort-area="750"
+      data-spaces-sort-bed="1"
+      data-spaces-sort-bath="1"
+      data-spaces-sort-price="1500"
+      data-spaces-available="true"
+      data-spaces-unavailable="false"
+      data-spaces-soonest="Now"
+      data-spaces-sort-date="1700000000"
+      data-spaces-plan-id="10"
+      data-spaces-sort-plan-name="Studio"
+      data-spaces-obj="unit"
+      data-spaces-community="TestCommunity"
+      data-spaces-asset="1"
+      data-spaces-href="/unit/${code}"
+      data-spaces-inventory-href="/inventory/${code}"
+    >
+      <img src="http://example.com/${code}.jpg" />
+    </article>
+  `).join('\n');
+
+  return `
+    <html><body>
+      <section class="spaces__tab-unit">
+        ${articles}
+      </section>
+    </body></html>
+  `;
+}
+
+// ============================================================
+// Test: recordScraperRun
+// ============================================================
+describe('recordScraperRun', () => {
+  it('should insert a run record into scraper_runs collection', async () => {
+    const runData = {
+      jobId: 'test-job-001',
+      trigger: 'manual',
+      status: 'success',
+      startedAt: new Date().toISOString(),
+      completedAt: new Date().toISOString(),
+      duration: 1234,
+      unitsProcessed: 10,
+      pricesInserted: 8,
+      errors: []
+    };
+
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(db, runData, mockLogger);
+
+    expect(result).toBeTruthy();
+    expect(result.insertedId).toBeTruthy();
+
+    // Verify it was inserted
+    const saved = await db.collection('scraper_runs').findOne({ jobId: 'test-job-001' });
+    expect(saved).toBeTruthy();
+    expect(saved.status).toBe('success');
+    expect(saved.recordedAt).toBeInstanceOf(Date);
+  });
+
+  it('should not throw when insert fails', async () => {
+    // Pass null db to cause an error
+    const runData = { jobId: 'test-fail', status: 'success' };
+
+    // Should not throw
+    const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+    const result = await recordScraperRun(null, runData, mockLogger);
+    expect(result).toBeNull();
+    expect(mockLogger.error).toHaveBeenCalled();
+  });
+});
+
+// ============================================================
+// Test: runScrape - Full workflow
+// ============================================================
+describe('runScrape', () => {
+  it('should complete full workflow with mocked dependencies', async () => {
+    const html = createSampleHtml(['UNIT-A', 'UNIT-B']);
+
+    const result = await runScrape(db, {
+      trigger: 'manual',
+      jobId: 'test-full-workflow',
+      htmlContent: html
+    });
+
+    expect(result).toBeTruthy();
+    expect(result.jobId).toBe('test-full-workflow');
+    expect(result.status).toBe('success');
+    expect(result.trigger).toBe('manual');
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.errors).toEqual([]);
+  });
+
+  it('should return correct result structure with jobId, status, metrics', async () => {
+    const html = createSampleHtml(['UNIT-X']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-structure',
+      htmlContent: html
+    });
+
+    // Verify all expected fields exist
+    expect(result).toHaveProperty('jobId', 'test-structure');
+    expect(result).toHaveProperty('trigger', 'manual');
+    expect(result).toHaveProperty('dryRun', false);
+    expect(result).toHaveProperty('status', 'success');
+    expect(result).toHaveProperty('startedAt');
+    expect(result).toHaveProperty('completedAt');
+    expect(result).toHaveProperty('duration');
+    expect(result).toHaveProperty('unitsProcessed');
+    expect(result).toHaveProperty('pricesInserted');
+    expect(result).toHaveProperty('newUnitsCount');
+    expect(result).toHaveProperty('rentedUnitsCount');
+    expect(result).toHaveProperty('staleUnitsCount');
+    expect(result).toHaveProperty('errors');
+
+    // Verify types
+    expect(typeof result.duration).toBe('number');
+    expect(result.duration).toBeGreaterThanOrEqual(0);
+    expect(typeof result.startedAt).toBe('string');
+    expect(typeof result.completedAt).toBe('string');
+    expect(Array.isArray(result.errors)).toBe(true);
+  });
+
+  it('should generate a jobId when not provided', async () => {
+    const html = createSampleHtml(['UNIT-GEN']);
+
+    const result = await runScrape(db, {
+      htmlContent: html
+    });
+
+    expect(result.jobId).toBeTruthy();
+    expect(typeof result.jobId).toBe('string');
+    // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+    expect(result.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
+  });
+
+  // ============================================================
+  // Test: dryRun option
+  // ============================================================
+  it('should skip DB writes when dryRun is true', async () => {
+    const html = createSampleHtml(['DRY-A', 'DRY-B']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-dry-run',
+      htmlContent: html,
+      dryRun: true
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.dryRun).toBe(true);
+    expect(result.unitsProcessed).toBe(2);
+    expect(result.pricesInserted).toBe(0);
+    expect(result.staleUnitsCount).toBe(0);
+
+    // Verify no units were written to the units collection
+    const unitsCount = await db.collection('units_scraper').countDocuments();
+    expect(unitsCount).toBe(0);
+
+    // Verify no prices were written
+    const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
+    expect(pricesCount).toBe(0);
+  });
+
+  // ============================================================
+  // Test: htmlContent option
+  // ============================================================
+  it('should use provided HTML instead of fetching when htmlContent is given', async () => {
+    const html = createSampleHtml(['HTML-1', 'HTML-2', 'HTML-3']);
+
+    // Mock axios.get to track if HTTP fetch is attempted
+    const axios = require('axios');
+    const axiosSpy = jest.spyOn(axios, 'get');
+
+    const result = await runScrape(db, {
+      jobId: 'test-html-content',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(3);
+    // axios.get should NOT have been called since we provided htmlContent
+    expect(axiosSpy).not.toHaveBeenCalled();
+  });
+
+  // ============================================================
+  // Test: Records scraper run on success
+  // ============================================================
+  it('should record scraper run to history on success', async () => {
+    const html = createSampleHtml(['REC-A']);
+
+    await runScrape(db, {
+      jobId: 'test-record-success',
+      htmlContent: html
+    });
+
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-success' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('success');
+    expect(runRecord.recordedAt).toBeInstanceOf(Date);
+    expect(runRecord.unitsProcessed).toBe(1);
+  });
+
+  // ============================================================
+  // Test: Records scraper run on failure
+  // ============================================================
+  it('should record scraper run to history on failure', async () => {
+    const html = createSampleHtml(['FAIL-A']);
+
+    // Create a db proxy that throws on bulkWrite (used by upsertUnits)
+    // but allows other operations (like scraper_runs insertOne) to pass through
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'units_scraper') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Database connection lost');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-record-failure',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Database connection lost');
+
+    // The run should still be recorded (via the real db passed through for scraper_runs)
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-record-failure' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+    expect(runRecord.errors).toContain('Database connection lost');
+  });
+
+  // ============================================================
+  // Test: Catches errors from fetchPage
+  // ============================================================
+  it('should catch and handle errors from fetchPage', async () => {
+    // Mock axios.get to throw a network error (fetchPage uses axios internally)
+    const axios = require('axios');
+    jest.spyOn(axios, 'get').mockRejectedValue(
+      new Error('Network timeout')
+    );
+
+    const result = await runScrape(db, {
+      jobId: 'test-fetch-error'
+      // No htmlContent, so it will call fetchPage which uses axios
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Network timeout');
+    expect(result.unitsProcessed).toBe(0);
+  });
+
+  // ============================================================
+  // Test: Catches errors from database operations
+  // ============================================================
+  it('should catch and handle errors from database operations', async () => {
+    const html = createSampleHtml(['DB-ERR']);
+
+    // Create a db proxy that throws on the prices collection bulkWrite
+    // Use Proxy to properly delegate all methods to the real collection
+    const faultyDb = {
+      collection: (name) => {
+        const realCollection = db.collection(name);
+        if (name === 'unit_prices_scraper') {
+          return new Proxy(realCollection, {
+            get(target, prop) {
+              if (prop === 'bulkWrite') {
+                return async () => {
+                  throw new Error('Write concern timeout');
+                };
+              }
+              const value = target[prop];
+              if (typeof value === 'function') {
+                return value.bind(target);
+              }
+              return value;
+            }
+          });
+        }
+        return realCollection;
+      }
+    };
+
+    const result = await runScrape(faultyDb, {
+      jobId: 'test-db-error',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.errors).toContain('Write concern timeout');
+
+    // Run should still be recorded despite error
+    const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-db-error' });
+    expect(runRecord).toBeTruthy();
+    expect(runRecord.status).toBe('failed');
+  });
+
+  // ============================================================
+  // Test: Calculates newUnitsCount and rentedUnitsCount correctly
+  // ============================================================
+  it('should calculate newUnitsCount and rentedUnitsCount correctly', async () => {
+    // First, simulate yesterday's data by inserting price records for yesterday
+    const today = new Date();
+    const yesterday = new Date(today);
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const yesterdayStr = yesterday.toISOString().split('T')[0];
+
+    // Yesterday had units: OLD-A, OLD-B, OLD-C
+    await db.collection('unit_prices_scraper').insertMany([
+      { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
+      { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
+      { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
+    ]);
+
+    // Today's scrape has: OLD-A, OLD-B, NEW-D (OLD-C is gone / rented)
+    const html = createSampleHtml(['OLD-A', 'OLD-B', 'NEW-D']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-calc-changes',
+      htmlContent: html
+    });
+
+    expect(result.status).toBe('success');
+    // NEW-D is new (not in yesterday's data)
+    expect(result.newUnitsCount).toBe(1);
+    // OLD-C was in yesterday's data but not in today's scrape
+    expect(result.rentedUnitsCount).toBe(1);
+    expect(result.unitsProcessed).toBe(3);
+  });
+
+  // ============================================================
+  // Test: Default trigger is 'manual'
+  // ============================================================
+  it('should default trigger to manual', async () => {
+    const html = createSampleHtml(['DEF-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-default-trigger',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('manual');
+  });
+
+  // ============================================================
+  // Test: Supports scheduled trigger
+  // ============================================================
+  it('should support scheduled trigger', async () => {
+    const html = createSampleHtml(['SCHED-A']);
+
+    const result = await runScrape(db, {
+      jobId: 'test-scheduled-trigger',
+      trigger: 'scheduled',
+      htmlContent: html
+    });
+
+    expect(result.trigger).toBe('scheduled');
+  });
+
+  // ============================================================
+  // Test: Handles empty HTML (no units found)
+  // ============================================================
+  it('should handle HTML with no units gracefully', async () => {
+    const emptyHtml = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+
+    const result = await runScrape(db, {
+      jobId: 'test-empty-html',
+      htmlContent: emptyHtml
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.unitsProcessed).toBe(0);
+    expect(result.errors).toContain('No units found in HTML');
+  });
+});
+
+// ============================================================
+// Test: sanitizeError - Error message sanitization
+// ============================================================
+describe('sanitizeError', () => {
+  let sanitizeError;
+
+  beforeAll(() => {
+    ({ sanitizeError } = require('../../services/scraperService'));
+  });
+
+  // ----------------------------------------------------------
+  // File path sanitization
+  // ----------------------------------------------------------
+  describe('file path sanitization', () => {
+    it('should remove Unix absolute file paths from error messages', () => {
+      const error = new Error('ENOENT: no such file or directory, open /home/user/app/config.json');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/home\/user/);
+      expect(sanitized.message).toContain('ENOENT');
+    });
+
+    it('should remove /var paths from error messages', () => {
+      const error = new Error('Failed to read /var/app/current/data/secrets.yml');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/var\/app/);
+      expect(sanitized.message).toContain('Failed to read');
+    });
+
+    it('should remove Windows-style file paths from error messages', () => {
+      const error = new Error('Cannot find module C:\\Users\\admin\\project\\node_modules\\secret');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/C:\\Users/);
+      expect(sanitized.message).toContain('Cannot find module');
+    });
+
+    it('should remove /tmp and /usr paths from error messages', () => {
+      const error = new Error('Error loading /tmp/scraper-cache/data.bin and /usr/local/lib/node.so');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/\/tmp\//);
+      expect(sanitized.message).not.toMatch(/\/usr\//);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // MongoDB connection string sanitization
+  // ----------------------------------------------------------
+  describe('connection string sanitization', () => {
+    it('should redact mongodb:// connection strings', () => {
+      const error = new Error('Connection failed: mongodb://admin:s3cretP4ss@db.example.com:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('s3cretP4ss');
+      expect(sanitized.message).not.toContain('admin:');
+      expect(sanitized.message).toContain('Connection failed');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact mongodb+srv:// connection strings', () => {
+      const error = new Error('Timeout connecting to mongodb+srv://user:password123@cluster0.abc.mongodb.net/mydb');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('password123');
+      expect(sanitized.message).not.toContain('user:');
+      expect(sanitized.message).toContain('Timeout connecting to');
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+
+    it('should redact connection string without credentials', () => {
+      const error = new Error('Cannot connect to mongodb://localhost:27017/apartments');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toMatch(/mongodb:\/\/localhost/);
+      expect(sanitized.message).toContain('[REDACTED_CONNECTION_STRING]');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Credential / secret sanitization
+  // ----------------------------------------------------------
+  describe('credential sanitization', () => {
+    it('should redact common environment variable patterns', () => {
+      const error = new Error('Invalid API_KEY=sk-abc123xyz or SECRET_TOKEN=bearer-9876');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('sk-abc123xyz');
+      expect(sanitized.message).not.toContain('bearer-9876');
+    });
+
+    it('should redact password patterns', () => {
+      const error = new Error('Auth failed with password=MyS3cret!');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).not.toContain('MyS3cret!');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Error type preservation
+  // ----------------------------------------------------------
+  describe('error type preservation', () => {
+    it('should preserve the error type/name', () => {
+      const error = new TypeError('Cannot read properties of undefined at /home/user/app/server.js:42');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TypeError');
+    });
+
+    it('should preserve custom error names', () => {
+      const error = new Error('Timeout at /var/app/scraper.js:100');
+      error.name = 'TimeoutError';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('TimeoutError');
+    });
+
+    it('should preserve error name for RangeError', () => {
+      const error = new RangeError('Maximum call stack size exceeded');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.name).toBe('RangeError');
+    });
+  });
+
+  // ----------------------------------------------------------
+  // General description preserved for debugging
+  // ----------------------------------------------------------
+  describe('general description preservation', () => {
+    it('should preserve a useful general description', () => {
+      const error = new Error('Network timeout after 30000ms');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('Network timeout after 30000ms');
+    });
+
+    it('should preserve error description when no sensitive data present', () => {
+      const error = new Error('Request failed with status code 500');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toBe('Request failed with status code 500');
+    });
+
+    it('should return a useful message even after heavy sanitization', () => {
+      const error = new Error('ECONNREFUSED mongodb://root:pass@host:27017 at /home/user/node_modules/mongodb/lib/connection.js:123');
+      const sanitized = sanitizeError(error);
+      expect(sanitized.message).toContain('ECONNREFUSED');
+      expect(sanitized.message.length).toBeGreaterThan(5);
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Stack trace sanitization
+  // ----------------------------------------------------------
+  describe('stack trace removal', () => {
+    it('should remove file paths from stack traces', () => {
+      const error = new Error('Something failed');
+      error.stack = 'Error: Something failed\n    at Object.<anonymous> (/home/user/app/services/scraperService.js:42:10)\n    at Module._compile (/usr/lib/node_modules/node/internal/modules/cjs/loader.js:1078:30)';
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).not.toMatch(/\/home\/user/);
+      expect(sanitized.stack).not.toMatch(/\/usr\/lib/);
+    });
+
+    it('should handle errors without stack trace', () => {
+      const error = new Error('No stack');
+      error.stack = undefined;
+      const sanitized = sanitizeError(error);
+      expect(sanitized.stack).toBeUndefined();
+    });
+  });
+
+  // ----------------------------------------------------------
+  // Integration: sanitization applied before recordScraperRun()
+  // ----------------------------------------------------------
+  describe('sanitization before recordScraperRun()', () => {
+    it('should store sanitized error message in scraper_runs on failure', async () => {
+      const html = createSampleHtml(['SANITIZE-A']);
+
+      // Create a db proxy that throws an error containing sensitive info
+      const sensitiveError = new Error(
+        'MongoServerError: connection to mongodb://admin:SuperSecret@db.prod.internal:27017/apartments failed at /home/deploy/app/node_modules/mongodb/lib/connection.js:370'
+      );
+      sensitiveError.name = 'MongoServerError';
+
+      const faultyDb = {
+        collection: (name) => {
+          const realCollection = db.collection(name);
+          if (name === 'units_scraper') {
+            return new Proxy(realCollection, {
+              get(target, prop) {
+                if (prop === 'bulkWrite') {
+                  return async () => { throw sensitiveError; };
+                }
+                const value = target[prop];
+                if (typeof value === 'function') {
+                  return value.bind(target);
+                }
+                return value;
+              }
+            });
+          }
+          return realCollection;
+        }
+      };
+
+      const result = await runScrape(faultyDb, {
+        jobId: 'test-sanitized-error',
+        htmlContent: html
+      });
+
+      expect(result.status).toBe('failed');
+
+      // Verify the error stored in result.errors is sanitized
+      const errorMsg = result.errors[0];
+      expect(errorMsg).not.toContain('SuperSecret');
+      expect(errorMsg).not.toContain('admin:');
+      expect(errorMsg).not.toContain('/home/deploy/');
+      expect(errorMsg).toContain('MongoServerError');
+
+      // Verify the error stored in scraper_runs is sanitized
+      const runRecord = await db.collection('scraper_runs').findOne({ jobId: 'test-sanitized-error' });
+      expect(runRecord).toBeTruthy();
+      expect(runRecord.status).toBe('failed');
+      const storedError = runRecord.errors[0];
+      expect(storedError).not.toContain('SuperSecret');
+      expect(storedError).not.toContain('admin:');
+      expect(storedError).not.toContain('/home/deploy/');
+    });
+  });
+});
+
+// ============================================================
+// Test: createScraperIndexes
+// ============================================================
+describe('createScraperIndexes', () => {
+  const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
+
+  beforeEach(() => {
+    mockLogger.info.mockClear();
+    mockLogger.warn.mockClear();
+    mockLogger.error.mockClear();
+  });
+
+  it('should create the status_startedAt compound index', async () => {
+    await createScraperIndexes(db, mockLogger);
+
+    const indexes = await db.collection('scraper_runs').indexes();
+    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
+    expect(statusIndex).toBeTruthy();
+    expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
+  });
+
+  it('should create the startedAt_desc index', async () => {
+    await createScraperIndexes(db, mockLogger);
+
+    const indexes = await db.collection('scraper_runs').indexes();
+    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
+    expect(startedAtIndex).toBeTruthy();
+    expect(startedAtIndex.key).toEqual({ startedAt: -1 });
+  });
+
+  it('should log success after creating indexes', async () => {
+    await createScraperIndexes(db, mockLogger);
+
+    expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
+  });
+
+  it('should be idempotent (no error on second call)', async () => {
+    await createScraperIndexes(db, mockLogger);
+
+    // Calling again should not throw
+    await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
+
+    // Indexes should still exist
+    const indexes = await db.collection('scraper_runs').indexes();
+    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
+    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
+    expect(statusIndex).toBeTruthy();
+    expect(startedAtIndex).toBeTruthy();
+  });
+
+  it('should handle database errors gracefully', async () => {
+    // Pass an object that will throw when collection() is called
+    const faultyDb = {
+      collection: () => {
+        throw new Error('Connection lost');
+      }
+    };
+
+    await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
+  });
+});

__tests__/scraper/serverIntegration.test.js

@@ -0,0 +1,177 @@
+/**
+ * Tests for scraper integration in server.js startup
+ *
+ * Covers:
+ * - Server initializes scraper scheduler on startup
+ * - Server does not crash if scheduler init fails
+ * - Indexes are created before scheduler starts
+ * - Scheduler not initialized when SCRAPER_ENABLED=false
+ *
+ * Strategy: Instead of importing server.js directly (which starts Express and
+ * connects to MongoDB), we replicate the scraper initialization logic from
+ * connectToMongoDB() and test it with mocked dependencies. This verifies the
+ * integration contract without needing a full server boot.
+ */
+
+const { MongoClient } = require('mongodb');
+
+describe('Server Scraper Integration', () => {
+  let connection;
+  let db;
+
+  beforeAll(async () => {
+    const uri = process.env.MONGO_URI;
+    connection = await MongoClient.connect(uri);
+    db = connection.db('apartments_server_integration_test');
+  });
+
+  afterAll(async () => {
+    if (connection) {
+      await connection.close();
+    }
+  });
+
+  describe('scraper initialization during startup', () => {
+    let mockCreateScraperIndexes;
+    let mockInitializeScheduler;
+    let mockCreateLogger;
+    let mockLogger;
+
+    beforeEach(() => {
+      mockLogger = {
+        info: jest.fn(),
+        warn: jest.fn(),
+        error: jest.fn(),
+        debug: jest.fn()
+      };
+      mockCreateLogger = jest.fn(() => mockLogger);
+      mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
+      mockInitializeScheduler = jest.fn();
+    });
+
+    /**
+     * Replicates the scraper initialization logic from server.js connectToMongoDB().
+     * This is the exact pattern used in production:
+     *
+     *   try {
+     *     const scraperLogger = createLogger('server-init');
+     *     await createScraperIndexes(db, scraperLogger);
+     *     if (scraperConfig.SCRAPER_ENABLED) {
+     *       initializeScheduler(db);
+     *     }
+     *   } catch (error) {
+     *     console.error('Scraper initialization failed (server continues):', error.message);
+     *   }
+     */
+    async function runScraperInit(testDb, config) {
+      try {
+        const scraperLogger = mockCreateLogger('server-init');
+        await mockCreateScraperIndexes(testDb, scraperLogger);
+        if (config.SCRAPER_ENABLED) {
+          mockInitializeScheduler(testDb);
+        }
+        return { success: true };
+      } catch (error) {
+        return { success: false, error: error.message };
+      }
+    }
+
+    it('should initialize scraper scheduler on startup when enabled', async () => {
+      const config = { SCRAPER_ENABLED: true };
+
+      const result = await runScraperInit(db, config);
+
+      expect(result.success).toBe(true);
+      expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
+      expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
+    });
+
+    it('should not crash if createScraperIndexes throws an error', async () => {
+      mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
+      const config = { SCRAPER_ENABLED: true };
+
+      const result = await runScraperInit(db, config);
+
+      // The server should continue (error is caught, not thrown)
+      expect(result.success).toBe(false);
+      expect(result.error).toBe('Index creation failed');
+      // initializeScheduler should NOT have been called since the error occurred before it
+      expect(mockInitializeScheduler).not.toHaveBeenCalled();
+    });
+
+    it('should create indexes before initializing the scheduler', async () => {
+      const callOrder = [];
+      mockCreateScraperIndexes.mockImplementation(async () => {
+        callOrder.push('createScraperIndexes');
+      });
+      mockInitializeScheduler.mockImplementation(() => {
+        callOrder.push('initializeScheduler');
+      });
+
+      const config = { SCRAPER_ENABLED: true };
+      await runScraperInit(db, config);
+
+      expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
+    });
+
+    it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
+      const config = { SCRAPER_ENABLED: false };
+
+      const result = await runScraperInit(db, config);
+
+      expect(result.success).toBe(true);
+      expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
+      expect(mockInitializeScheduler).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('server.js imports and exports verification', () => {
+    it('should be able to import createScraperIndexes from scraperService', () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      expect(typeof createScraperIndexes).toBe('function');
+    });
+
+    it('should be able to import initializeScheduler from scraperJob', () => {
+      const { initializeScheduler } = require('../../jobs/scraperJob');
+      expect(typeof initializeScheduler).toBe('function');
+    });
+
+    it('should be able to import registerSignalHandlers from scraperJob', () => {
+      const { registerSignalHandlers } = require('../../jobs/scraperJob');
+      expect(typeof registerSignalHandlers).toBe('function');
+    });
+
+    it('should be able to import SCRAPER_ENABLED from scraper config', () => {
+      const scraperConfig = require('../../config/scraper');
+      expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
+    });
+  });
+
+  describe('createScraperIndexes error handling with real db', () => {
+    it('should handle index creation gracefully with a valid db', async () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      const { createLogger } = require('../../services/scraperLogger');
+      const logger = createLogger('test-server-init');
+
+      // Should not throw - creates indexes on the test db
+      await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
+    });
+
+    it('should throw when given an invalid db object', async () => {
+      const { createScraperIndexes } = require('../../services/scraperService');
+      const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
+
+      // Passing null should throw (simulates what the try/catch in server.js protects against)
+      await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
+    });
+  });
+
+  describe('registerSignalHandlers', () => {
+    it('should register SIGTERM and SIGINT handlers without throwing', () => {
+      const { registerSignalHandlers } = require('../../jobs/scraperJob');
+
+      // Should not throw when called
+      expect(() => registerSignalHandlers()).not.toThrow();
+    });
+  });
+});

__tests__/scraper/updateDailySummary.test.js

@@ -0,0 +1,561 @@
+/**
+ * Tests for updateDailySummary() daily summary aggregation
+ *
+ * Covers:
+ * - Upsert by date key (YYYY-MM-DD)
+ * - Summary includes new_units, rented_units, stale_units arrays
+ * - Count fields are calculated correctly
+ * - net_change = newUnits.length - rentedUnits.length
+ * - total_available_today is set correctly
+ * - total_available_yesterday fetched from previous day
+ * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+ * - turnover_rate is 0 when yesterdayTotal is 0
+ * - Error handling when updateOne fails
+ * - Logging of summary update details
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require updateDailySummary after implementation
+let updateDailySummary;
+
+let mongoServer;
+let client;
+let db;
+
+const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ updateDailySummary } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('updateDailySummary', () => {
+  // ---------------------------------------------------------------
+  // 1. Upsert by date key
+  // ---------------------------------------------------------------
+  describe('upsert by date key', () => {
+    it('should insert a new summary when no document exists for the date', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 2,
+        totalAvailable: 15
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have upserted (created new)
+      expect(result.upsertedCount).toBe(1);
+
+      // Verify the document in the database
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc).not.toBeNull();
+      expect(doc.date).toBe('2026-02-05');
+    });
+
+    it('should update an existing summary for the same date (idempotent upsert)', async () => {
+      const logger = createMockLogger();
+
+      // Insert initial summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-05',
+        new_units: ['A101'],
+        rented_units: [],
+        total_available_today: 10
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 12
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have matched and modified (not upserted)
+      expect(result.matchedCount).toBe(1);
+
+      // Verify updated values
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
+      expect(doc.total_available_today).toBe(12);
+
+      // Verify only one document exists for this date
+      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Summary includes new_units, rented_units, stale_units arrays
+  // ---------------------------------------------------------------
+  describe('summary arrays', () => {
+    it('should store new_units, rented_units, and stale_units arrays', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202', 'B203'],
+        staleUnitsCount: 3,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102']);
+      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
+      expect(doc.stale_units).toEqual([]);
+    });
+
+    it('should handle empty arrays for new_units and rented_units', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 25
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual([]);
+      expect(doc.rented_units).toEqual([]);
+      expect(doc.stale_units).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Count fields are calculated correctly
+  // ---------------------------------------------------------------
+  describe('count fields', () => {
+    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 5,
+        totalAvailable: 30
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(3);
+      expect(doc.rented_units_count).toBe(1);
+      expect(doc.stale_units_count).toBe(5);
+    });
+
+    it('should set all count fields to 0 when arrays are empty', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(0);
+      expect(doc.rented_units_count).toBe(0);
+      expect(doc.stale_units_count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. net_change = newUnits.length - rentedUnits.length
+  // ---------------------------------------------------------------
+  describe('net_change calculation', () => {
+    it('should calculate positive net_change when more units added than rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(2); // 3 - 1 = 2
+    });
+
+    it('should calculate negative net_change when more units rented than added', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
+    });
+
+    it('should calculate zero net_change when equal new and rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. total_available_today is set correctly
+  // ---------------------------------------------------------------
+  describe('total_available_today', () => {
+    it('should store the totalAvailable value as total_available_today', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 42
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(42);
+    });
+
+    it('should handle totalAvailable of 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 0
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. total_available_yesterday fetched from previous day
+  // ---------------------------------------------------------------
+  describe('total_available_yesterday', () => {
+    it('should fetch total_available_today from previous day summary', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 35,
+        new_units: [],
+        rented_units: []
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 35
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(35);
+    });
+
+    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(0);
+    });
+
+    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
+      const logger = createMockLogger();
+
+      // Insert March 31 summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-03-31',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-04-01',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 48
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
+      expect(doc.total_available_yesterday).toBe(50);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+  // ---------------------------------------------------------------
+  describe('turnover_rate calculation', () => {
+    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 50 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
+        staleUnitsCount: 0,
+        totalAvailable: 46
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 5 rented / 50 yesterday = 10%
+      expect(doc.turnover_rate).toBe(10);
+    });
+
+    it('should round turnover_rate to 2 decimal places', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 30 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 30
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 28
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
+      expect(doc.turnover_rate).toBe(6.67);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. turnover_rate is 0 when yesterdayTotal is 0
+  // ---------------------------------------------------------------
+  describe('turnover_rate edge cases', () => {
+    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 0 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 0
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Error handling when updateOne fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw and log error when updateOne fails', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      // Create a mock db that throws on updateOne
+      const mockCollection = {
+        findOne: jest.fn().mockResolvedValue(null),
+        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(
+        updateDailySummary(mockDb, summaryData, logger)
+      ).rejects.toThrow('Write concern timeout');
+
+      // Verify error was logged
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to update daily summary',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Logging of summary update details
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log summary update details on success', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 18
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Daily summary updated',
+        expect.objectContaining({
+          date: '2026-02-05',
+          newUnits: 2,
+          rentedUnits: 1,
+          totalAvailable: 18
+        })
+      );
+    });
+
+    it('should include timestamp field in the stored summary', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.timestamp).toBeDefined();
+      expect(typeof doc.timestamp).toBe('string');
+      // Verify it is a valid ISO timestamp
+      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
+    });
+  });
+});

__tests__/scraper/upsertUnits.test.js

@@ -0,0 +1,498 @@
+/**
+ * Tests for upsertUnits() bulk operation
+ *
+ * Covers:
+ * - bulkWrite is called with correct updateOne operations
+ * - upsert: true is set for all operations
+ * - $set updates last_scraped and data_source fields
+ * - $setOnInsert sets first_seen for new units
+ * - Empty units array logs warning and returns early
+ * - ordered: false is set for parallel execution
+ * - Error handling when bulkWrite fails
+ * - Logging of matched, modified, and upserted counts
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// We will require upsertUnits after implementation
+let upsertUnits;
+
+let mongoServer;
+let client;
+let db;
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ upsertUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean the units collection before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('upsertUnits', () => {
+  const UNITS_COLLECTION = 'units_scraper';
+
+  // ---------------------------------------------------------------
+  // 1. bulkWrite is called with correct updateOne operations
+  // ---------------------------------------------------------------
+  describe('bulkWrite operations', () => {
+    it('should create updateOne operations for each unit', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'A101', price: 1200, bed_count: 1, area: 500 },
+        { unit_code: 'B202', price: 1500, bed_count: 2, area: 750 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // Both units should be upserted (new inserts)
+      expect(result.upsertedCount).toBe(2);
+
+      // Verify documents exist in the collection
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(2);
+
+      const unitCodes = docs.map(d => d.unit_code).sort();
+      expect(unitCodes).toEqual(['A101', 'B202']);
+    });
+
+    it('should filter by unit_code as the unique key', async () => {
+      const logger = createMockLogger();
+
+      // Insert a unit first
+      const units = [{ unit_code: 'A101', price: 1200, bed_count: 1 }];
+      await upsertUnits(db, units, logger);
+
+      // Upsert again with updated price
+      const updatedUnits = [{ unit_code: 'A101', price: 1400, bed_count: 1 }];
+      const result = await upsertUnits(db, updatedUnits, logger);
+
+      // Should match and modify, not insert new
+      expect(result.matchedCount).toBe(1);
+      expect(result.upsertedCount).toBe(0);
+
+      // Verify only one document exists
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      expect(docs).toHaveLength(1);
+      expect(docs[0].price).toBe(1400);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. upsert: true is set for all operations
+  // ---------------------------------------------------------------
+  describe('upsert behavior', () => {
+    it('should insert new units when they do not exist (upsert: true)', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'NEW-001', price: 1100 },
+        { unit_code: 'NEW-002', price: 1200 },
+        { unit_code: 'NEW-003', price: 1300 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(3);
+
+      // Verify all three are in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+
+    it('should update existing units without creating duplicates', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'X100', price: 900 }], logger);
+
+      // Second upsert with same unit_code
+      await upsertUnits(db, [{ unit_code: 'X100', price: 950 }], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. $set updates last_scraped and data_source fields
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set last_scraped timestamp on every upsert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      const units = [{ unit_code: 'T100', price: 1000 }];
+      await upsertUnits(db, units, logger);
+
+      const afterTime = new Date().toISOString();
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T100' });
+      expect(doc.last_scraped).toBeDefined();
+      expect(typeof doc.last_scraped).toBe('string');
+      // The last_scraped should be between before and after timestamps
+      expect(doc.last_scraped >= beforeTime).toBe(true);
+      expect(doc.last_scraped <= afterTime).toBe(true);
+    });
+
+    it('should set data_source to "web_scraper"', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'T200', price: 1100 }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T200' });
+      expect(doc.data_source).toBe('web_scraper');
+    });
+
+    it('should update last_scraped on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 800 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const firstScraped = doc1.last_scraped;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert
+      await upsertUnits(db, [{ unit_code: 'T300', price: 850 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T300' });
+      const secondScraped = doc2.last_scraped;
+
+      expect(secondScraped > firstScraped).toBe(true);
+    });
+
+    it('should spread all unit fields into the $set operation', async () => {
+      const logger = createMockLogger();
+      const units = [{
+        unit_code: 'T400',
+        price: 1500,
+        bed_count: 2,
+        bath_count: 1.5,
+        area: 900,
+        floor: 3,
+        plan_name: 'Luxury',
+        community: 'Tower A'
+      }];
+
+      await upsertUnits(db, units, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'T400' });
+      expect(doc.price).toBe(1500);
+      expect(doc.bed_count).toBe(2);
+      expect(doc.bath_count).toBe(1.5);
+      expect(doc.area).toBe(900);
+      expect(doc.floor).toBe(3);
+      expect(doc.plan_name).toBe('Luxury');
+      expect(doc.community).toBe('Tower A');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. $setOnInsert sets first_seen for new units
+  // ---------------------------------------------------------------
+  describe('$setOnInsert for first_seen', () => {
+    it('should set first_seen on initial insert', async () => {
+      const logger = createMockLogger();
+      const beforeTime = new Date().toISOString();
+
+      await upsertUnits(db, [{ unit_code: 'F100', price: 1000 }], logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F100' });
+      expect(doc.first_seen).toBeDefined();
+      expect(typeof doc.first_seen).toBe('string');
+      expect(doc.first_seen >= beforeTime).toBe(true);
+    });
+
+    it('should NOT update first_seen on subsequent upserts', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+      const originalFirstSeen = doc1.first_seen;
+
+      // Small delay to ensure different timestamp
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      // Second upsert (update)
+      await upsertUnits(db, [{ unit_code: 'F200', price: 1100 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F200' });
+
+      // first_seen should remain unchanged
+      expect(doc2.first_seen).toBe(originalFirstSeen);
+    });
+
+    it('should set different first_seen for different units inserted at different times', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [{ unit_code: 'F300', price: 1000 }], logger);
+      const doc1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F300' });
+
+      await new Promise(resolve => setTimeout(resolve, 10));
+
+      await upsertUnits(db, [{ unit_code: 'F400', price: 1200 }], logger);
+      const doc2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'F400' });
+
+      // Both should have first_seen, but different values
+      expect(doc1.first_seen).toBeDefined();
+      expect(doc2.first_seen).toBeDefined();
+      expect(doc2.first_seen > doc1.first_seen).toBe(true);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Empty units array logs warning and returns early
+  // ---------------------------------------------------------------
+  describe('empty units array', () => {
+    it('should log a warning when units array is empty', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      expect(logger.warn).toHaveBeenCalledWith('No units to upsert');
+    });
+
+    it('should return early with zero counts for empty array', async () => {
+      const logger = createMockLogger();
+
+      const result = await upsertUnits(db, [], logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(result.upsertedCount).toBe(0);
+    });
+
+    it('should not perform any database writes for empty array', async () => {
+      const logger = createMockLogger();
+
+      await upsertUnits(db, [], logger);
+
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. ordered: false is set for parallel execution
+  // ---------------------------------------------------------------
+  describe('ordered: false for parallel execution', () => {
+    it('should successfully process all units even if one has a conflict', async () => {
+      const logger = createMockLogger();
+
+      // Insert multiple units - ordered: false means all operations execute
+      // even if some fail. We verify by checking all valid units are present.
+      const units = [
+        { unit_code: 'P100', price: 1000 },
+        { unit_code: 'P200', price: 1100 },
+        { unit_code: 'P300', price: 1200 },
+        { unit_code: 'P400', price: 1300 },
+        { unit_code: 'P500', price: 1400 }
+      ];
+
+      const result = await upsertUnits(db, units, logger);
+
+      expect(result.upsertedCount).toBe(5);
+
+      // All 5 units should be in the database
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(5);
+    });
+
+    it('should handle a mix of new inserts and existing updates', async () => {
+      const logger = createMockLogger();
+
+      // First, insert some units
+      await upsertUnits(db, [
+        { unit_code: 'M100', price: 900 },
+        { unit_code: 'M200', price: 1000 }
+      ], logger);
+
+      // Now upsert a mix of existing and new units
+      const mixedUnits = [
+        { unit_code: 'M100', price: 950 },  // existing - update
+        { unit_code: 'M200', price: 1050 }, // existing - update
+        { unit_code: 'M300', price: 1100 }, // new - insert
+      ];
+
+      const result = await upsertUnits(db, mixedUnits, logger);
+
+      expect(result.matchedCount).toBe(2);
+      expect(result.upsertedCount).toBe(1);
+
+      // Total should be 3 documents
+      const count = await db.collection(UNITS_COLLECTION).countDocuments();
+      expect(count).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Error handling when bulkWrite fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E100', price: 1000 }];
+
+      // Create a mock db that throws on bulkWrite
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when bulkWrite fails', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E200', price: 1000 }];
+
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      try {
+        await upsertUnits(mockDb, units, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to upsert units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'E300', price: 1000 }];
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        bulkWrite: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(upsertUnits(mockDb, units, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Logging of matched, modified, and upserted counts
+  // ---------------------------------------------------------------
+  describe('logging of result counts', () => {
+    it('should log matched, modified, and upserted counts on success', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L100', price: 1000 },
+        { unit_code: 'L200', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: expect.any(Number),
+          modified: expect.any(Number),
+          upserted: expect.any(Number)
+        })
+      );
+    });
+
+    it('should log correct counts for new inserts', async () => {
+      const logger = createMockLogger();
+      const units = [
+        { unit_code: 'L300', price: 1000 },
+        { unit_code: 'L400', price: 1100 }
+      ];
+
+      await upsertUnits(db, units, logger);
+
+      // For new inserts: matched=0, modified=0, upserted=2
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 0,
+          modified: 0,
+          upserted: 2
+        })
+      );
+    });
+
+    it('should log correct counts for updates to existing units', async () => {
+      const logger = createMockLogger();
+
+      // First insert
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1000 }], logger);
+
+      // Reset mock to capture only the second call
+      logger.info.mockClear();
+
+      // Update existing
+      await upsertUnits(db, [{ unit_code: 'L500', price: 1100 }], logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Units upserted',
+        expect.objectContaining({
+          matched: 1,
+          modified: 1,
+          upserted: 0
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the bulkWrite result object', async () => {
+      const logger = createMockLogger();
+      const units = [{ unit_code: 'R100', price: 1000 }];
+
+      const result = await upsertUnits(db, units, logger);
+
+      // bulkWrite result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('upsertedCount');
+    });
+  });
+});