Implement parseUnits() HTML parsing with cheerio

- Extract apartment unit data from section.spaces__tab-unit container
- Parse all data-spaces-* attributes from article elements
- Deduplicate units by unit_code with warning logging
- Handle missing container gracefully (returns empty array)
- Add comprehensive test suite with HTML fixture (22 tests)

__tests__/scraper/fixtures/sample-listing.html

@@ -0,0 +1,144 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Country Club Towers - Available Units</title>
+</head>
+<body>
+  <div class="spaces-container">
+    <section class="spaces__tab-unit">
+      <!-- Unit 1: Fully populated with all attributes and an image -->
+      <article
+        data-spaces-id="1001"
+        data-spaces-unit="CCT-101"
+        data-spaces-unit-id="5001"
+        data-spaces-unit-floor="1"
+        data-spaces-sort-area="750"
+        data-spaces-sort-bed="1"
+        data-spaces-sort-bath="1"
+        data-spaces-sort-price="1450"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="2026-03-01"
+        data-spaces-sort-date="20260301"
+        data-spaces-plan-id="201"
+        data-spaces-sort-plan-name="Alpine"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-101"
+        data-spaces-inventory-href="/inventory/CCT-101"
+        data-spaces-specials-content="First month free!"
+      >
+        <div class="unit-card">
+          <img src="https://example.com/images/unit-101.jpg" alt="Unit 101" />
+          <h3>Unit CCT-101</h3>
+          <p>1 Bed / 1 Bath - $1,450/mo</p>
+        </div>
+      </article>
+
+      <!-- Unit 2: Different floor plan, no specials, with data-src image -->
+      <article
+        data-spaces-id="1002"
+        data-spaces-unit="CCT-205"
+        data-spaces-unit-id="5002"
+        data-spaces-unit-floor="2"
+        data-spaces-sort-area="950"
+        data-spaces-sort-bed="2"
+        data-spaces-sort-bath="1.5"
+        data-spaces-sort-price="1850"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="2026-02-15"
+        data-spaces-sort-date="20260215"
+        data-spaces-plan-id="202"
+        data-spaces-sort-plan-name="Birchwood"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-205"
+        data-spaces-inventory-href="/inventory/CCT-205"
+        data-spaces-specials-content=""
+      >
+        <div class="unit-card">
+          <img data-src="https://example.com/images/unit-205.jpg" alt="Unit 205" />
+          <h3>Unit CCT-205</h3>
+        </div>
+      </article>
+
+      <!-- Unit 3: Studio with some missing attributes -->
+      <article
+        data-spaces-id="1003"
+        data-spaces-unit="CCT-310"
+        data-spaces-unit-id="5003"
+        data-spaces-unit-floor="3"
+        data-spaces-sort-area="500"
+        data-spaces-sort-bed="0"
+        data-spaces-sort-bath="1"
+        data-spaces-sort-price="1100"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-soonest="Available Now"
+        data-spaces-sort-date="20260201"
+        data-spaces-plan-id="203"
+        data-spaces-sort-plan-name="Cedar"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-310"
+        data-spaces-inventory-href="/inventory/CCT-310"
+      >
+        <div class="unit-card">
+          <h3>Unit CCT-310</h3>
+          <p>Studio - $1,100/mo</p>
+        </div>
+      </article>
+
+      <!-- Unit 4: Unavailable unit -->
+      <article
+        data-spaces-id="1004"
+        data-spaces-unit="CCT-415"
+        data-spaces-unit-id="5004"
+        data-spaces-unit-floor="4"
+        data-spaces-sort-area="1200"
+        data-spaces-sort-bed="3"
+        data-spaces-sort-bath="2"
+        data-spaces-sort-price="2500"
+        data-spaces-available="false"
+        data-spaces-unavailable="true"
+        data-spaces-soonest=""
+        data-spaces-sort-date=""
+        data-spaces-plan-id="204"
+        data-spaces-sort-plan-name="Dogwood"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+        data-spaces-href="/units/CCT-415"
+        data-spaces-inventory-href="/inventory/CCT-415"
+        data-spaces-specials-content=""
+      >
+        <div class="unit-card">
+          <img src="https://example.com/images/unit-415.jpg" alt="Unit 415" />
+          <h3>Unit CCT-415</h3>
+        </div>
+      </article>
+
+      <!-- Unit 5: Minimal attributes - edge case -->
+      <article
+        data-spaces-id="1005"
+        data-spaces-unit="CCT-520"
+        data-spaces-unit-id="5005"
+        data-spaces-sort-price="1300"
+        data-spaces-available="true"
+        data-spaces-unavailable="false"
+        data-spaces-obj="unit"
+        data-spaces-community="Country Club Towers"
+        data-spaces-asset="100"
+      >
+        <div class="unit-card">
+          <h3>Unit CCT-520</h3>
+        </div>
+      </article>
+    </section>
+  </div>
+</body>
+</html>

__tests__/scraper/parseUnits.test.js

@@ -0,0 +1,360 @@
+/**
+ * Tests for parseUnits() - HTML parsing with cheerio
+ *
+ * Covers:
+ * - Extraction of all data-spaces-* attributes
+ * - Image URL extraction from nested img elements
+ * - Container detection (section.spaces__tab-unit)
+ * - Empty / missing container handling
+ * - Deduplication by unit_code
+ * - Logging behavior (info, warn, error)
+ * - Missing attribute handling
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseUnits } = require('../../services/scraperService');
+
+const fixturesDir = path.join(__dirname, 'fixtures');
+
+// Helper: create a mock logger
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+describe('parseUnits', () => {
+  let sampleHtml;
+  let mockLogger;
+
+  beforeAll(() => {
+    sampleHtml = fs.readFileSync(
+      path.join(fixturesDir, 'sample-listing.html'),
+      'utf-8'
+    );
+  });
+
+  beforeEach(() => {
+    mockLogger = createMockLogger();
+  });
+
+  // ---------------------------------------------------------------
+  // 1. Container detection
+  // ---------------------------------------------------------------
+  describe('container detection', () => {
+    it('should find section.spaces__tab-unit container in valid HTML', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+
+      // Should not log an error about missing container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+      expect(units.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array when section container is missing', () => {
+      const html = '<html><body><div>No units here</div></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+
+    it('should return empty array for empty HTML string', () => {
+      const units = parseUnits('', mockLogger);
+
+      expect(units).toEqual([]);
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    it('should return empty array when container exists but has no articles', () => {
+      const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
+      const units = parseUnits(html, mockLogger);
+
+      expect(units).toEqual([]);
+      // Container found, no error about container
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Extraction of all data-spaces-* attributes
+  // ---------------------------------------------------------------
+  describe('attribute extraction', () => {
+    it('should extract all expected attributes from a fully populated article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
+
+      expect(unit1).toBeDefined();
+
+      // Core identifiers
+      expect(unit1.id).toBe('1001');
+      expect(unit1.unit_code).toBe('CCT-101');
+      expect(unit1.unit_id).toBe('5001');
+
+      // Physical attributes
+      expect(unit1.floor).toBe('1');
+      expect(unit1.area).toBe('750');
+      expect(unit1.bed_count).toBe('1');
+      expect(unit1.bath_count).toBe('1');
+
+      // Pricing
+      expect(unit1.price).toBe('1450');
+
+      // Availability
+      expect(unit1.available).toBe('true');
+      expect(unit1.unavailable).toBe('false');
+      expect(unit1.soonest).toBe('2026-03-01');
+      expect(unit1.date_available).toBe('20260301');
+
+      // Plan information
+      expect(unit1.plan_id).toBe('201');
+      expect(unit1.plan_name).toBe('Alpine');
+
+      // Property information
+      expect(unit1.obj_type).toBe('unit');
+      expect(unit1.community).toBe('Country Club Towers');
+      expect(unit1.asset).toBe('100');
+
+      // URLs
+      expect(unit1.href).toBe('/units/CCT-101');
+      expect(unit1.inventory_href).toBe('/inventory/CCT-101');
+
+      // Specials
+      expect(unit1.specials_content).toBe('First month free!');
+    });
+
+    it('should extract correct count of units from the fixture', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      expect(units).toHaveLength(5);
+    });
+
+    it('should return raw string values without type conversion', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit = units.find(u => u.unit_code === 'CCT-101');
+
+      // All values should be strings (raw extraction, no conversion)
+      expect(typeof unit.id).toBe('string');
+      expect(typeof unit.price).toBe('string');
+      expect(typeof unit.floor).toBe('string');
+      expect(typeof unit.available).toBe('string');
+      expect(typeof unit.bed_count).toBe('string');
+      expect(typeof unit.bath_count).toBe('string');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Image URL extraction
+  // ---------------------------------------------------------------
+  describe('image URL extraction', () => {
+    it('should extract image_url from img src attribute', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
+
+      expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
+    });
+
+    it('should fall back to data-src when src is not present', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
+
+      expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
+    });
+
+    it('should not set image_url when no img element exists', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
+
+      expect(unit3.image_url).toBeUndefined();
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Missing attributes
+  // ---------------------------------------------------------------
+  describe('missing attributes', () => {
+    it('should return undefined for attributes not present on the article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      // Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
+      // soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
+
+      expect(unit5).toBeDefined();
+      expect(unit5.floor).toBeUndefined();
+      expect(unit5.area).toBeUndefined();
+      expect(unit5.bed_count).toBeUndefined();
+      expect(unit5.bath_count).toBeUndefined();
+      expect(unit5.soonest).toBeUndefined();
+      expect(unit5.date_available).toBeUndefined();
+      expect(unit5.plan_id).toBeUndefined();
+      expect(unit5.plan_name).toBeUndefined();
+      expect(unit5.href).toBeUndefined();
+      expect(unit5.inventory_href).toBeUndefined();
+      expect(unit5.specials_content).toBeUndefined();
+    });
+
+    it('should still extract the attributes that are present on a sparse article', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
+
+      expect(unit5.id).toBe('1005');
+      expect(unit5.unit_code).toBe('CCT-520');
+      expect(unit5.unit_id).toBe('5005');
+      expect(unit5.price).toBe('1300');
+      expect(unit5.available).toBe('true');
+      expect(unit5.obj_type).toBe('unit');
+      expect(unit5.community).toBe('Country Club Towers');
+      expect(unit5.asset).toBe('100');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Deduplication
+  // ---------------------------------------------------------------
+  describe('deduplication', () => {
+    it('should deduplicate units with the same unit_code', () => {
+      // Build HTML with duplicate articles
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+      const codes = units.map(u => u.unit_code);
+
+      // Should have 2 unique unit codes
+      expect(units).toHaveLength(2);
+      expect(new Set(codes).size).toBe(codes.length);
+    });
+
+    it('should keep the first occurrence when deduplicating', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // First occurrence (id=1, price=1000) should be kept
+      expect(units[0].id).toBe('1');
+      expect(units[0].price).toBe('1000');
+    });
+
+    it('should log a warning when duplicates are removed', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
+          <article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
+        </section>
+        </body></html>
+      `;
+
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Duplicate units removed',
+        expect.objectContaining({
+          original: 3,
+          deduplicated: 2
+        })
+      );
+    });
+
+    it('should not log a warning when there are no duplicates', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      // The fixture has no duplicates, so no warn about duplicates
+      const warnCalls = mockLogger.warn.mock.calls;
+      const duplicateWarns = warnCalls.filter(
+        call => call[0] === 'Duplicate units removed'
+      );
+      expect(duplicateWarns).toHaveLength(0);
+    });
+
+    it('should filter out articles with no unit_code during deduplication', () => {
+      const html = `
+        <html><body>
+        <section class="spaces__tab-unit">
+          <article data-spaces-id="1" data-spaces-sort-price="1000"></article>
+          <article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
+        </section>
+        </body></html>
+      `;
+
+      const units = parseUnits(html, mockLogger);
+
+      // Article without unit_code should be filtered out
+      expect(units).toHaveLength(1);
+      expect(units[0].unit_code).toBe('VALID-1');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging behavior
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of parsed units', () => {
+      parseUnits(sampleHtml, mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Units parsed',
+        expect.objectContaining({ count: 5 })
+      );
+    });
+
+    it('should log error when container is not found', () => {
+      const html = '<html><body><p>Nothing here</p></body></html>';
+      parseUnits(html, mockLogger);
+
+      expect(mockLogger.error).toHaveBeenCalledTimes(1);
+      expect(mockLogger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Container section.spaces__tab-unit not found'),
+        expect.any(Object)
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Multiple units with varied data
+  // ---------------------------------------------------------------
+  describe('varied unit data', () => {
+    it('should handle unavailable units correctly', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit4 = units.find(u => u.unit_code === 'CCT-415');
+
+      expect(unit4.available).toBe('false');
+      expect(unit4.unavailable).toBe('true');
+      expect(unit4.soonest).toBe('');
+      expect(unit4.date_available).toBe('');
+    });
+
+    it('should handle half bath counts', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
+
+      expect(unit2.bath_count).toBe('1.5');
+    });
+
+    it('should handle studio units (bed_count = 0)', () => {
+      const units = parseUnits(sampleHtml, mockLogger);
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
+
+      expect(unit3.bed_count).toBe('0');
+    });
+  });
+});

services/scraperService.js

@@ -6,6 +6,7 @@
  */
 
 const axios = require('axios');
+const cheerio = require('cheerio');
 const config = require('../config/scraper');
 
 /**
@@ -106,8 +107,100 @@ async function fetchPage(url, logger) {
   throw lastError;
 }
 
+/**
+ * Parse unit data from HTML using cheerio
+ * @param {string} html - HTML content
+ * @param {Object} logger - Logger instance
+ * @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
+ */
+function parseUnits(html, logger) {
+  const $ = cheerio.load(html);
+  const units = [];
+
+  // Find the main container
+  const container = $('section.spaces__tab-unit');
+
+  if (container.length === 0) {
+    logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
+    return [];
+  }
+
+  // Extract each article element
+  container.find('article').each((index, article) => {
+    const $article = $(article);
+
+    const unit = {
+      // Core identifiers
+      id: $article.attr('data-spaces-id'),
+      unit_code: $article.attr('data-spaces-unit'),
+      unit_id: $article.attr('data-spaces-unit-id'),
+
+      // Physical attributes
+      floor: $article.attr('data-spaces-unit-floor'),
+      area: $article.attr('data-spaces-sort-area'),
+      bed_count: $article.attr('data-spaces-sort-bed'),
+      bath_count: $article.attr('data-spaces-sort-bath'),
+
+      // Pricing
+      price: $article.attr('data-spaces-sort-price'),
+
+      // Availability
+      available: $article.attr('data-spaces-available'),
+      unavailable: $article.attr('data-spaces-unavailable'),
+      soonest: $article.attr('data-spaces-soonest'),
+      date_available: $article.attr('data-spaces-sort-date'),
+
+      // Plan information
+      plan_id: $article.attr('data-spaces-plan-id'),
+      plan_name: $article.attr('data-spaces-sort-plan-name'),
+
+      // Property information
+      obj_type: $article.attr('data-spaces-obj'),
+      community: $article.attr('data-spaces-community'),
+      asset: $article.attr('data-spaces-asset'),
+
+      // URLs
+      href: $article.attr('data-spaces-href'),
+      inventory_href: $article.attr('data-spaces-inventory-href'),
+
+      // Specials
+      specials_content: $article.attr('data-spaces-specials-content')
+    };
+
+    // Extract image URL from nested element if present
+    const imgElement = $article.find('img').first();
+    if (imgElement.length > 0) {
+      unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
+    }
+
+    units.push(unit);
+  });
+
+  logger.info('Units parsed', { count: units.length });
+
+  // Deduplicate by unit_code (in case of duplicate articles)
+  const seen = new Set();
+  const deduplicated = units.filter(unit => {
+    if (!unit.unit_code || seen.has(unit.unit_code)) {
+      return false;
+    }
+    seen.add(unit.unit_code);
+    return true;
+  });
+
+  if (deduplicated.length < units.length) {
+    logger.warn('Duplicate units removed', {
+      original: units.length,
+      deduplicated: deduplicated.length
+    });
+  }
+
+  return deduplicated;
+}
+
 module.exports = {
   fetchPage,
+  parseUnits,
   // Export for testing
   isRetryableError,
   sleep