__tests__/scraper/dataConversion.test.js

@@ -0,0 +1,496 @@
+/**
+ * Tests for convertDataTypes() and its internal helper functions
+ *
+ * Covers:
+ * - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
+ * - parseInteger: valid integers, null/empty/invalid
+ * - parsePositiveInteger: treats 0 as null (for area)
+ * - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
+ * - parseFloat: "1.5" for half baths, null/empty/invalid
+ * - parseBoolean: "true"/"false"/"1"/"0", null/empty
+ * - trimString: trim whitespace, null for empty, null/undefined
+ * - convertDataTypes: full integration applying correct parser to each field
+ */
+
+const {
+  convertDataTypes,
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString
+} = require('../../services/scraperService');
+
+// ---------------------------------------------------------------
+// 1. parsePrice
+// ---------------------------------------------------------------
+describe('parsePrice', () => {
+  it('should parse "$1,234" format to 1234', () => {
+    expect(parsePrice('$1,234')).toBe(1234);
+  });
+
+  it('should parse plain numeric string', () => {
+    expect(parsePrice('1450')).toBe(1450);
+  });
+
+  it('should parse "$2,500.00" stripping non-numeric chars', () => {
+    expect(parsePrice('$2,500.00')).toBe(2500);
+  });
+
+  it('should return null for "Call for pricing"', () => {
+    expect(parsePrice('Call for pricing')).toBeNull();
+  });
+
+  it('should return null for "Contact us"', () => {
+    expect(parsePrice('Contact us')).toBeNull();
+  });
+
+  it('should return null for "Please inquire"', () => {
+    expect(parsePrice('Please inquire')).toBeNull();
+  });
+
+  it('should return null for negative values', () => {
+    // After stripping non-numeric chars, "-1500" becomes "1500"
+    // but if someone passes a numeric -1500 directly, check that too
+    expect(parsePrice('-')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(parsePrice(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePrice('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parsePrice(undefined)).toBeNull();
+  });
+
+  it('should return null for completely non-numeric string', () => {
+    expect(parsePrice('abc')).toBeNull();
+  });
+
+  it('should handle price of 0', () => {
+    expect(parsePrice('0')).toBe(0);
+  });
+
+  it('should handle large prices', () => {
+    expect(parsePrice('$15,500')).toBe(15500);
+  });
+});
+
+// ---------------------------------------------------------------
+// 2. parseInteger
+// ---------------------------------------------------------------
+describe('parseInteger', () => {
+  it('should parse valid integer string', () => {
+    expect(parseInteger('42')).toBe(42);
+  });
+
+  it('should parse "0" to 0', () => {
+    expect(parseInteger('0')).toBe(0);
+  });
+
+  it('should parse negative integer string', () => {
+    expect(parseInteger('-5')).toBe(-5);
+  });
+
+  it('should return null for null', () => {
+    expect(parseInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseInteger('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseInteger(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseInteger('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseInteger('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseInteger('Infinity')).toBeNull();
+  });
+
+  it('should truncate float strings to integer', () => {
+    expect(parseInteger('3.7')).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------
+// 3. parsePositiveInteger
+// ---------------------------------------------------------------
+describe('parsePositiveInteger', () => {
+  it('should parse positive integer normally', () => {
+    expect(parsePositiveInteger('750')).toBe(750);
+  });
+
+  it('should treat 0 as null (for area field)', () => {
+    expect(parsePositiveInteger('0')).toBeNull();
+  });
+
+  it('should return null for null input', () => {
+    expect(parsePositiveInteger(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parsePositiveInteger('')).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parsePositiveInteger('abc')).toBeNull();
+  });
+
+  it('should handle negative values (parsed as negative int, not positive)', () => {
+    expect(parsePositiveInteger('-5')).toBe(-5);
+  });
+});
+
+// ---------------------------------------------------------------
+// 4. parseIntegerOrString
+// ---------------------------------------------------------------
+describe('parseIntegerOrString', () => {
+  it('should return integer when value is cleanly parseable', () => {
+    expect(parseIntegerOrString('5001')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric value', () => {
+    expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
+  });
+
+  it('should return trimmed string when value has non-numeric suffix', () => {
+    expect(parseIntegerOrString('123abc')).toBe('123abc');
+  });
+
+  it('should return null for null', () => {
+    expect(parseIntegerOrString(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseIntegerOrString('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseIntegerOrString(undefined)).toBeNull();
+  });
+
+  it('should trim whitespace before checking', () => {
+    expect(parseIntegerOrString('  5001  ')).toBe(5001);
+  });
+
+  it('should return trimmed string for non-numeric with whitespace', () => {
+    expect(parseIntegerOrString('  abc  ')).toBe('abc');
+  });
+});
+
+// ---------------------------------------------------------------
+// 5. parseFloatValue
+// ---------------------------------------------------------------
+describe('parseFloatValue', () => {
+  it('should parse "1.5" for half baths', () => {
+    expect(parseFloatValue('1.5')).toBe(1.5);
+  });
+
+  it('should parse "2.0" to 2', () => {
+    expect(parseFloatValue('2.0')).toBe(2.0);
+  });
+
+  it('should parse integer string as float', () => {
+    expect(parseFloatValue('3')).toBe(3);
+  });
+
+  it('should return null for null', () => {
+    expect(parseFloatValue(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseFloatValue('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseFloatValue(undefined)).toBeNull();
+  });
+
+  it('should return null for non-numeric string', () => {
+    expect(parseFloatValue('abc')).toBeNull();
+  });
+
+  it('should return null for NaN', () => {
+    expect(parseFloatValue('NaN')).toBeNull();
+  });
+
+  it('should return null for Infinity', () => {
+    expect(parseFloatValue('Infinity')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 6. parseBoolean
+// ---------------------------------------------------------------
+describe('parseBoolean', () => {
+  it('should parse "true" to true', () => {
+    expect(parseBoolean('true')).toBe(true);
+  });
+
+  it('should parse "false" to false', () => {
+    expect(parseBoolean('false')).toBe(false);
+  });
+
+  it('should parse "1" to true', () => {
+    expect(parseBoolean('1')).toBe(true);
+  });
+
+  it('should parse "0" to false', () => {
+    expect(parseBoolean('0')).toBe(false);
+  });
+
+  it('should parse "TRUE" (case insensitive) to true', () => {
+    expect(parseBoolean('TRUE')).toBe(true);
+  });
+
+  it('should parse "False" (case insensitive) to false', () => {
+    expect(parseBoolean('False')).toBe(false);
+  });
+
+  it('should pass through boolean true', () => {
+    expect(parseBoolean(true)).toBe(true);
+  });
+
+  it('should pass through boolean false', () => {
+    expect(parseBoolean(false)).toBe(false);
+  });
+
+  it('should return null for null', () => {
+    expect(parseBoolean(null)).toBeNull();
+  });
+
+  it('should return null for empty string', () => {
+    expect(parseBoolean('')).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(parseBoolean(undefined)).toBeNull();
+  });
+
+  it('should return null for unrecognized string', () => {
+    expect(parseBoolean('yes')).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------
+// 7. trimString
+// ---------------------------------------------------------------
+describe('trimString', () => {
+  it('should trim leading and trailing whitespace', () => {
+    expect(trimString('  hello  ')).toBe('hello');
+  });
+
+  it('should return the string as-is if no whitespace', () => {
+    expect(trimString('hello')).toBe('hello');
+  });
+
+  it('should return null for empty string', () => {
+    expect(trimString('')).toBeNull();
+  });
+
+  it('should return null for whitespace-only string', () => {
+    expect(trimString('   ')).toBeNull();
+  });
+
+  it('should return null for null', () => {
+    expect(trimString(null)).toBeNull();
+  });
+
+  it('should return null for undefined', () => {
+    expect(trimString(undefined)).toBeNull();
+  });
+
+  it('should convert non-string to string then trim', () => {
+    expect(trimString(123)).toBe('123');
+  });
+});
+
+// ---------------------------------------------------------------
+// 8. convertDataTypes - Full integration test
+// ---------------------------------------------------------------
+describe('convertDataTypes', () => {
+  it('should apply correct parser to each field', () => {
+    const rawUnit = {
+      id: '1001',
+      unit_code: 'CCT-101',
+      unit_id: '5001',
+      floor: '1',
+      area: '750',
+      bed_count: '1',
+      bath_count: '1.5',
+      price: '1450',
+      available: 'true',
+      unavailable: 'false',
+      soonest: '2026-03-01',
+      date_available: '20260301',
+      plan_id: '201',
+      plan_name: 'Alpine',
+      obj_type: 'unit',
+      community: 'Country Club Towers',
+      asset: '100',
+      href: '/units/CCT-101',
+      inventory_href: '/inventory/CCT-101',
+      image_url: 'https://example.com/images/unit-101.jpg',
+      specials_content: 'First month free!'
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    // Integer fields
+    expect(converted.id).toBe(1001);
+    expect(converted.floor).toBe(1);
+    expect(converted.bed_count).toBe(1);
+    expect(converted.plan_id).toBe(201);
+    expect(converted.asset).toBe(100);
+    expect(converted.date_available).toBe(20260301);
+
+    // parseIntegerOrString
+    expect(converted.unit_id).toBe(5001);
+
+    // parsePositiveInteger (area)
+    expect(converted.area).toBe(750);
+
+    // Float field (bath_count)
+    expect(converted.bath_count).toBe(1.5);
+
+    // Price field
+    expect(converted.price).toBe(1450);
+
+    // Boolean fields
+    expect(converted.available).toBe(true);
+    expect(converted.unavailable).toBe(false);
+
+    // String fields (trimmed)
+    expect(converted.unit_code).toBe('CCT-101');
+    expect(converted.plan_name).toBe('Alpine');
+    expect(converted.soonest).toBe('2026-03-01');
+    expect(converted.obj_type).toBe('unit');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.href).toBe('/units/CCT-101');
+    expect(converted.inventory_href).toBe('/inventory/CCT-101');
+    expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
+    expect(converted.specials_content).toBe('First month free!');
+  });
+
+  it('should preserve string fields without converting them', () => {
+    const rawUnit = {
+      unit_code: 'CCT-205',
+      plan_name: '  Birch  ',
+      soonest: 'Now',
+      obj_type: 'unit',
+      community: '  Country Club Towers  ',
+      href: '/units/CCT-205',
+      inventory_href: '/inventory/CCT-205',
+      image_url: 'https://example.com/img.jpg',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.unit_code).toBe('CCT-205');
+    expect(converted.plan_name).toBe('Birch');
+    expect(converted.community).toBe('Country Club Towers');
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle a unit with all null/undefined/empty values', () => {
+    const rawUnit = {
+      id: null,
+      unit_code: undefined,
+      unit_id: '',
+      floor: null,
+      area: '',
+      bed_count: undefined,
+      bath_count: null,
+      price: '',
+      available: null,
+      unavailable: undefined,
+      soonest: '',
+      date_available: null,
+      plan_id: undefined,
+      plan_name: '',
+      obj_type: null,
+      community: undefined,
+      asset: '',
+      href: null,
+      inventory_href: undefined,
+      image_url: '',
+      specials_content: null
+    };
+
+    const converted = convertDataTypes(rawUnit);
+
+    expect(converted.id).toBeNull();
+    expect(converted.unit_code).toBeNull();
+    expect(converted.unit_id).toBeNull();
+    expect(converted.floor).toBeNull();
+    expect(converted.area).toBeNull();
+    expect(converted.bed_count).toBeNull();
+    expect(converted.bath_count).toBeNull();
+    expect(converted.price).toBeNull();
+    expect(converted.available).toBeNull();
+    expect(converted.unavailable).toBeNull();
+    expect(converted.soonest).toBeNull();
+    expect(converted.date_available).toBeNull();
+    expect(converted.plan_id).toBeNull();
+    expect(converted.plan_name).toBeNull();
+    expect(converted.obj_type).toBeNull();
+    expect(converted.community).toBeNull();
+    expect(converted.asset).toBeNull();
+    expect(converted.href).toBeNull();
+    expect(converted.inventory_href).toBeNull();
+    expect(converted.image_url).toBeNull();
+    expect(converted.specials_content).toBeNull();
+  });
+
+  it('should handle area=0 as null (parsePositiveInteger)', () => {
+    const rawUnit = { area: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.area).toBeNull();
+  });
+
+  it('should handle bed_count=0 as valid (studio)', () => {
+    const rawUnit = { bed_count: '0' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bed_count).toBe(0);
+  });
+
+  it('should handle "Call for pricing" in price field', () => {
+    const rawUnit = { price: 'Call for pricing' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBeNull();
+  });
+
+  it('should handle "$1,234" price format', () => {
+    const rawUnit = { price: '$1,234' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.price).toBe(1234);
+  });
+
+  it('should handle unit_id as string identifier', () => {
+    const rawUnit = { unit_id: 'UNIT-ABC' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.unit_id).toBe('UNIT-ABC');
+  });
+
+  it('should handle bath_count with half baths', () => {
+    const rawUnit = { bath_count: '1.5' };
+    const converted = convertDataTypes(rawUnit);
+    expect(converted.bath_count).toBe(1.5);
+  });
+});

services/scraperService.js

@@ -198,10 +198,189 @@ function parseUnits(html, logger) {
   return deduplicated;
 }
 
+// ============================================================
+// Data Type Conversion Helpers
+// ============================================================
+
+/**
+ * Parse string to integer, return null for invalid
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed integer or null
+ */
+function parseInteger(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Parse string to positive integer, return null for 0 or invalid.
+ * Used for fields like area where 0 means "not available".
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed positive integer or null
+ */
+function parsePositiveInteger(value) {
+  const parsed = parseInteger(value);
+  return parsed === 0 ? null : parsed;
+}
+
+/**
+ * Parse value that could be integer or string identifier.
+ * Returns integer if cleanly parseable, otherwise trimmed string.
+ * @param {*} value - Value to parse
+ * @returns {number|string|null} Parsed integer, trimmed string, or null
+ */
+function parseIntegerOrString(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = parseInt(value, 10);
+  // If it parses cleanly to an integer, return integer
+  if (!Number.isNaN(parsed) && String(parsed) === String(value).trim()) {
+    return parsed;
+  }
+  // Otherwise return as trimmed string
+  return String(value).trim();
+}
+
+/**
+ * Parse price, stripping non-numeric characters.
+ * Handles "$1,234" format and "Call for pricing" text.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed price or null
+ */
+function parsePrice(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+
+  // Check for "Call for pricing" or similar text
+  if (typeof value === 'string' && /call|contact|inquire/i.test(value)) {
+    return null;
+  }
+
+  // Strip non-numeric characters except decimal point
+  const cleaned = String(value).replace(/[^0-9.]/g, '');
+  const parsed = parseInt(cleaned, 10);
+
+  if (Number.isNaN(parsed) || !Number.isFinite(parsed)) {
+    return null;
+  }
+
+  // Negative prices are invalid
+  if (parsed < 0) {
+    return null;
+  }
+
+  return parsed;
+}
+
+/**
+ * Parse boolean from string.
+ * Handles "true"/"false"/"1"/"0" and actual boolean values.
+ * @param {*} value - Value to parse
+ * @returns {boolean|null} Parsed boolean or null
+ */
+function parseBoolean(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  const str = String(value).toLowerCase().trim();
+  if (str === 'true' || str === '1') return true;
+  if (str === 'false' || str === '0') return false;
+  return null;
+}
+
+/**
+ * Parse float value.
+ * Named parseFloatValue to avoid shadowing the global parseFloat.
+ * @param {*} value - Value to parse
+ * @returns {number|null} Parsed float or null
+ */
+function parseFloatValue(value) {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isNaN(parsed) || !Number.isFinite(parsed) ? null : parsed;
+}
+
+/**
+ * Trim string, return null for empty.
+ * Converts non-string values to string before trimming.
+ * @param {*} value - Value to trim
+ * @returns {string|null} Trimmed string or null
+ */
+function trimString(value) {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  const trimmed = String(value).trim();
+  return trimmed === '' ? null : trimmed;
+}
+
+// ============================================================
+// Main Data Type Conversion Function
+// ============================================================
+
+/**
+ * Convert unit data types from strings to proper types.
+ * Applies the appropriate parser to each field based on its expected type.
+ * @param {Object} unit - Raw unit object with string values
+ * @returns {Object} Unit object with converted types
+ */
+function convertDataTypes(unit) {
+  return {
+    // Integer fields
+    id: parseInteger(unit.id),
+    unit_id: parseIntegerOrString(unit.unit_id),
+    floor: parseInteger(unit.floor),
+    area: parsePositiveInteger(unit.area),
+    bed_count: parseInteger(unit.bed_count),
+    plan_id: parseInteger(unit.plan_id),
+    asset: parseInteger(unit.asset),
+    date_available: parseInteger(unit.date_available),
+
+    // Float fields
+    bath_count: parseFloatValue(unit.bath_count),
+
+    // Price field (special handling)
+    price: parsePrice(unit.price),
+
+    // Boolean fields
+    available: parseBoolean(unit.available),
+    unavailable: parseBoolean(unit.unavailable),
+
+    // String fields (trim and preserve)
+    unit_code: trimString(unit.unit_code),
+    plan_name: trimString(unit.plan_name),
+    soonest: trimString(unit.soonest),
+    obj_type: trimString(unit.obj_type),
+    community: trimString(unit.community),
+    href: trimString(unit.href),
+    inventory_href: trimString(unit.inventory_href),
+    image_url: trimString(unit.image_url),
+    specials_content: trimString(unit.specials_content)
+  };
+}
+
 module.exports = {
   fetchPage,
   parseUnits,
-  // Export for testing
+  convertDataTypes,
+  // Export helpers for testing
+  parseInteger,
+  parsePositiveInteger,
+  parseIntegerOrString,
+  parsePrice,
+  parseBoolean,
+  parseFloatValue,
+  trimString,
   isRetryableError,
   sleep
 };