SCRAPE-5: Implement convertDataTypes() with type parsing helpers (#10)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
496
__tests__/scraper/dataConversion.test.js
Normal file
496
__tests__/scraper/dataConversion.test.js
Normal file
@ -0,0 +1,496 @@
|
||||
/**
|
||||
* Tests for convertDataTypes() and its internal helper functions
|
||||
*
|
||||
* Covers:
|
||||
* - parsePrice: "$1,234" format, "Call for pricing", negative values, null/empty
|
||||
* - parseInteger: valid integers, null/empty/invalid
|
||||
* - parsePositiveInteger: treats 0 as null (for area)
|
||||
* - parseIntegerOrString: int when cleanly parseable, trimmed string otherwise
|
||||
* - parseFloat: "1.5" for half baths, null/empty/invalid
|
||||
* - parseBoolean: "true"/"false"/"1"/"0", null/empty
|
||||
* - trimString: trim whitespace, null for empty, null/undefined
|
||||
* - convertDataTypes: full integration applying correct parser to each field
|
||||
*/
|
||||
|
||||
const {
|
||||
convertDataTypes,
|
||||
parseInteger,
|
||||
parsePositiveInteger,
|
||||
parseIntegerOrString,
|
||||
parsePrice,
|
||||
parseBoolean,
|
||||
parseFloatValue,
|
||||
trimString
|
||||
} = require('../../services/scraperService');
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. parsePrice
|
||||
// ---------------------------------------------------------------
|
||||
describe('parsePrice', () => {
|
||||
it('should parse "$1,234" format to 1234', () => {
|
||||
expect(parsePrice('$1,234')).toBe(1234);
|
||||
});
|
||||
|
||||
it('should parse plain numeric string', () => {
|
||||
expect(parsePrice('1450')).toBe(1450);
|
||||
});
|
||||
|
||||
it('should parse "$2,500.00" stripping non-numeric chars', () => {
|
||||
expect(parsePrice('$2,500.00')).toBe(2500);
|
||||
});
|
||||
|
||||
it('should return null for "Call for pricing"', () => {
|
||||
expect(parsePrice('Call for pricing')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for "Contact us"', () => {
|
||||
expect(parsePrice('Contact us')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for "Please inquire"', () => {
|
||||
expect(parsePrice('Please inquire')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for negative values', () => {
|
||||
// After stripping non-numeric chars, "-1500" becomes "1500"
|
||||
// but if someone passes a numeric -1500 directly, check that too
|
||||
expect(parsePrice('-')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parsePrice(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parsePrice('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parsePrice(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for completely non-numeric string', () => {
|
||||
expect(parsePrice('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle price of 0', () => {
|
||||
expect(parsePrice('0')).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle large prices', () => {
|
||||
expect(parsePrice('$15,500')).toBe(15500);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. parseInteger
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseInteger', () => {
|
||||
it('should parse valid integer string', () => {
|
||||
expect(parseInteger('42')).toBe(42);
|
||||
});
|
||||
|
||||
it('should parse "0" to 0', () => {
|
||||
expect(parseInteger('0')).toBe(0);
|
||||
});
|
||||
|
||||
it('should parse negative integer string', () => {
|
||||
expect(parseInteger('-5')).toBe(-5);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseInteger(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseInteger('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseInteger(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parseInteger('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for NaN', () => {
|
||||
expect(parseInteger('NaN')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for Infinity', () => {
|
||||
expect(parseInteger('Infinity')).toBeNull();
|
||||
});
|
||||
|
||||
it('should truncate float strings to integer', () => {
|
||||
expect(parseInteger('3.7')).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. parsePositiveInteger
|
||||
// ---------------------------------------------------------------
|
||||
describe('parsePositiveInteger', () => {
|
||||
it('should parse positive integer normally', () => {
|
||||
expect(parsePositiveInteger('750')).toBe(750);
|
||||
});
|
||||
|
||||
it('should treat 0 as null (for area field)', () => {
|
||||
expect(parsePositiveInteger('0')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null input', () => {
|
||||
expect(parsePositiveInteger(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parsePositiveInteger('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parsePositiveInteger('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle negative values (parsed as negative int, not positive)', () => {
|
||||
expect(parsePositiveInteger('-5')).toBe(-5);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. parseIntegerOrString
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseIntegerOrString', () => {
|
||||
it('should return integer when value is cleanly parseable', () => {
|
||||
expect(parseIntegerOrString('5001')).toBe(5001);
|
||||
});
|
||||
|
||||
it('should return trimmed string for non-numeric value', () => {
|
||||
expect(parseIntegerOrString('unit-abc')).toBe('unit-abc');
|
||||
});
|
||||
|
||||
it('should return trimmed string when value has non-numeric suffix', () => {
|
||||
expect(parseIntegerOrString('123abc')).toBe('123abc');
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseIntegerOrString(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseIntegerOrString('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseIntegerOrString(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should trim whitespace before checking', () => {
|
||||
expect(parseIntegerOrString(' 5001 ')).toBe(5001);
|
||||
});
|
||||
|
||||
it('should return trimmed string for non-numeric with whitespace', () => {
|
||||
expect(parseIntegerOrString(' abc ')).toBe('abc');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. parseFloatValue
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseFloatValue', () => {
|
||||
it('should parse "1.5" for half baths', () => {
|
||||
expect(parseFloatValue('1.5')).toBe(1.5);
|
||||
});
|
||||
|
||||
it('should parse "2.0" to 2', () => {
|
||||
expect(parseFloatValue('2.0')).toBe(2.0);
|
||||
});
|
||||
|
||||
it('should parse integer string as float', () => {
|
||||
expect(parseFloatValue('3')).toBe(3);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseFloatValue(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseFloatValue('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseFloatValue(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for non-numeric string', () => {
|
||||
expect(parseFloatValue('abc')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for NaN', () => {
|
||||
expect(parseFloatValue('NaN')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for Infinity', () => {
|
||||
expect(parseFloatValue('Infinity')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. parseBoolean
|
||||
// ---------------------------------------------------------------
|
||||
describe('parseBoolean', () => {
|
||||
it('should parse "true" to true', () => {
|
||||
expect(parseBoolean('true')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "false" to false', () => {
|
||||
expect(parseBoolean('false')).toBe(false);
|
||||
});
|
||||
|
||||
it('should parse "1" to true', () => {
|
||||
expect(parseBoolean('1')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "0" to false', () => {
|
||||
expect(parseBoolean('0')).toBe(false);
|
||||
});
|
||||
|
||||
it('should parse "TRUE" (case insensitive) to true', () => {
|
||||
expect(parseBoolean('TRUE')).toBe(true);
|
||||
});
|
||||
|
||||
it('should parse "False" (case insensitive) to false', () => {
|
||||
expect(parseBoolean('False')).toBe(false);
|
||||
});
|
||||
|
||||
it('should pass through boolean true', () => {
|
||||
expect(parseBoolean(true)).toBe(true);
|
||||
});
|
||||
|
||||
it('should pass through boolean false', () => {
|
||||
expect(parseBoolean(false)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(parseBoolean(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(parseBoolean('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(parseBoolean(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for unrecognized string', () => {
|
||||
expect(parseBoolean('yes')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. trimString
|
||||
// ---------------------------------------------------------------
|
||||
describe('trimString', () => {
|
||||
it('should trim leading and trailing whitespace', () => {
|
||||
expect(trimString(' hello ')).toBe('hello');
|
||||
});
|
||||
|
||||
it('should return the string as-is if no whitespace', () => {
|
||||
expect(trimString('hello')).toBe('hello');
|
||||
});
|
||||
|
||||
it('should return null for empty string', () => {
|
||||
expect(trimString('')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for whitespace-only string', () => {
|
||||
expect(trimString(' ')).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null', () => {
|
||||
expect(trimString(null)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for undefined', () => {
|
||||
expect(trimString(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('should convert non-string to string then trim', () => {
|
||||
expect(trimString(123)).toBe('123');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. convertDataTypes - Full integration test
|
||||
// ---------------------------------------------------------------
|
||||
describe('convertDataTypes', () => {
|
||||
it('should apply correct parser to each field', () => {
|
||||
const rawUnit = {
|
||||
id: '1001',
|
||||
unit_code: 'CCT-101',
|
||||
unit_id: '5001',
|
||||
floor: '1',
|
||||
area: '750',
|
||||
bed_count: '1',
|
||||
bath_count: '1.5',
|
||||
price: '1450',
|
||||
available: 'true',
|
||||
unavailable: 'false',
|
||||
soonest: '2026-03-01',
|
||||
date_available: '20260301',
|
||||
plan_id: '201',
|
||||
plan_name: 'Alpine',
|
||||
obj_type: 'unit',
|
||||
community: 'Country Club Towers',
|
||||
asset: '100',
|
||||
href: '/units/CCT-101',
|
||||
inventory_href: '/inventory/CCT-101',
|
||||
image_url: 'https://example.com/images/unit-101.jpg',
|
||||
specials_content: 'First month free!'
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
// Integer fields
|
||||
expect(converted.id).toBe(1001);
|
||||
expect(converted.floor).toBe(1);
|
||||
expect(converted.bed_count).toBe(1);
|
||||
expect(converted.plan_id).toBe(201);
|
||||
expect(converted.asset).toBe(100);
|
||||
expect(converted.date_available).toBe(20260301);
|
||||
|
||||
// parseIntegerOrString
|
||||
expect(converted.unit_id).toBe(5001);
|
||||
|
||||
// parsePositiveInteger (area)
|
||||
expect(converted.area).toBe(750);
|
||||
|
||||
// Float field (bath_count)
|
||||
expect(converted.bath_count).toBe(1.5);
|
||||
|
||||
// Price field
|
||||
expect(converted.price).toBe(1450);
|
||||
|
||||
// Boolean fields
|
||||
expect(converted.available).toBe(true);
|
||||
expect(converted.unavailable).toBe(false);
|
||||
|
||||
// String fields (trimmed)
|
||||
expect(converted.unit_code).toBe('CCT-101');
|
||||
expect(converted.plan_name).toBe('Alpine');
|
||||
expect(converted.soonest).toBe('2026-03-01');
|
||||
expect(converted.obj_type).toBe('unit');
|
||||
expect(converted.community).toBe('Country Club Towers');
|
||||
expect(converted.href).toBe('/units/CCT-101');
|
||||
expect(converted.inventory_href).toBe('/inventory/CCT-101');
|
||||
expect(converted.image_url).toBe('https://example.com/images/unit-101.jpg');
|
||||
expect(converted.specials_content).toBe('First month free!');
|
||||
});
|
||||
|
||||
it('should preserve string fields without converting them', () => {
|
||||
const rawUnit = {
|
||||
unit_code: 'CCT-205',
|
||||
plan_name: ' Birch ',
|
||||
soonest: 'Now',
|
||||
obj_type: 'unit',
|
||||
community: ' Country Club Towers ',
|
||||
href: '/units/CCT-205',
|
||||
inventory_href: '/inventory/CCT-205',
|
||||
image_url: 'https://example.com/img.jpg',
|
||||
specials_content: null
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
expect(converted.unit_code).toBe('CCT-205');
|
||||
expect(converted.plan_name).toBe('Birch');
|
||||
expect(converted.community).toBe('Country Club Towers');
|
||||
expect(converted.specials_content).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle a unit with all null/undefined/empty values', () => {
|
||||
const rawUnit = {
|
||||
id: null,
|
||||
unit_code: undefined,
|
||||
unit_id: '',
|
||||
floor: null,
|
||||
area: '',
|
||||
bed_count: undefined,
|
||||
bath_count: null,
|
||||
price: '',
|
||||
available: null,
|
||||
unavailable: undefined,
|
||||
soonest: '',
|
||||
date_available: null,
|
||||
plan_id: undefined,
|
||||
plan_name: '',
|
||||
obj_type: null,
|
||||
community: undefined,
|
||||
asset: '',
|
||||
href: null,
|
||||
inventory_href: undefined,
|
||||
image_url: '',
|
||||
specials_content: null
|
||||
};
|
||||
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
|
||||
expect(converted.id).toBeNull();
|
||||
expect(converted.unit_code).toBeNull();
|
||||
expect(converted.unit_id).toBeNull();
|
||||
expect(converted.floor).toBeNull();
|
||||
expect(converted.area).toBeNull();
|
||||
expect(converted.bed_count).toBeNull();
|
||||
expect(converted.bath_count).toBeNull();
|
||||
expect(converted.price).toBeNull();
|
||||
expect(converted.available).toBeNull();
|
||||
expect(converted.unavailable).toBeNull();
|
||||
expect(converted.soonest).toBeNull();
|
||||
expect(converted.date_available).toBeNull();
|
||||
expect(converted.plan_id).toBeNull();
|
||||
expect(converted.plan_name).toBeNull();
|
||||
expect(converted.obj_type).toBeNull();
|
||||
expect(converted.community).toBeNull();
|
||||
expect(converted.asset).toBeNull();
|
||||
expect(converted.href).toBeNull();
|
||||
expect(converted.inventory_href).toBeNull();
|
||||
expect(converted.image_url).toBeNull();
|
||||
expect(converted.specials_content).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle area=0 as null (parsePositiveInteger)', () => {
|
||||
const rawUnit = { area: '0' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.area).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle bed_count=0 as valid (studio)', () => {
|
||||
const rawUnit = { bed_count: '0' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.bed_count).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle "Call for pricing" in price field', () => {
|
||||
const rawUnit = { price: 'Call for pricing' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.price).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle "$1,234" price format', () => {
|
||||
const rawUnit = { price: '$1,234' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.price).toBe(1234);
|
||||
});
|
||||
|
||||
it('should handle unit_id as string identifier', () => {
|
||||
const rawUnit = { unit_id: 'UNIT-ABC' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.unit_id).toBe('UNIT-ABC');
|
||||
});
|
||||
|
||||
it('should handle bath_count with half baths', () => {
|
||||
const rawUnit = { bath_count: '1.5' };
|
||||
const converted = convertDataTypes(rawUnit);
|
||||
expect(converted.bath_count).toBe(1.5);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user