Merge dev: Node.js scraper migration + CI fix #32
144
__tests__/scraper/fixtures/sample-listing.html
Normal file
144
__tests__/scraper/fixtures/sample-listing.html
Normal file
@ -0,0 +1,144 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Country Club Towers - Available Units</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="spaces-container">
|
||||
<section class="spaces__tab-unit">
|
||||
<!-- Unit 1: Fully populated with all attributes and an image -->
|
||||
<article
|
||||
data-spaces-id="1001"
|
||||
data-spaces-unit="CCT-101"
|
||||
data-spaces-unit-id="5001"
|
||||
data-spaces-unit-floor="1"
|
||||
data-spaces-sort-area="750"
|
||||
data-spaces-sort-bed="1"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-sort-price="1450"
|
||||
data-spaces-available="true"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-soonest="2026-03-01"
|
||||
data-spaces-sort-date="20260301"
|
||||
data-spaces-plan-id="201"
|
||||
data-spaces-sort-plan-name="Alpine"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100"
|
||||
data-spaces-href="/units/CCT-101"
|
||||
data-spaces-inventory-href="/inventory/CCT-101"
|
||||
data-spaces-specials-content="First month free!"
|
||||
>
|
||||
<div class="unit-card">
|
||||
<img src="https://example.com/images/unit-101.jpg" alt="Unit 101" />
|
||||
<h3>Unit CCT-101</h3>
|
||||
<p>1 Bed / 1 Bath - $1,450/mo</p>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<!-- Unit 2: Different floor plan, no specials, with data-src image -->
|
||||
<article
|
||||
data-spaces-id="1002"
|
||||
data-spaces-unit="CCT-205"
|
||||
data-spaces-unit-id="5002"
|
||||
data-spaces-unit-floor="2"
|
||||
data-spaces-sort-area="950"
|
||||
data-spaces-sort-bed="2"
|
||||
data-spaces-sort-bath="1.5"
|
||||
data-spaces-sort-price="1850"
|
||||
data-spaces-available="true"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-soonest="2026-02-15"
|
||||
data-spaces-sort-date="20260215"
|
||||
data-spaces-plan-id="202"
|
||||
data-spaces-sort-plan-name="Birchwood"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100"
|
||||
data-spaces-href="/units/CCT-205"
|
||||
data-spaces-inventory-href="/inventory/CCT-205"
|
||||
data-spaces-specials-content=""
|
||||
>
|
||||
<div class="unit-card">
|
||||
<img data-src="https://example.com/images/unit-205.jpg" alt="Unit 205" />
|
||||
<h3>Unit CCT-205</h3>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<!-- Unit 3: Studio with some missing attributes -->
|
||||
<article
|
||||
data-spaces-id="1003"
|
||||
data-spaces-unit="CCT-310"
|
||||
data-spaces-unit-id="5003"
|
||||
data-spaces-unit-floor="3"
|
||||
data-spaces-sort-area="500"
|
||||
data-spaces-sort-bed="0"
|
||||
data-spaces-sort-bath="1"
|
||||
data-spaces-sort-price="1100"
|
||||
data-spaces-available="true"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-soonest="Available Now"
|
||||
data-spaces-sort-date="20260201"
|
||||
data-spaces-plan-id="203"
|
||||
data-spaces-sort-plan-name="Cedar"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100"
|
||||
data-spaces-href="/units/CCT-310"
|
||||
data-spaces-inventory-href="/inventory/CCT-310"
|
||||
>
|
||||
<div class="unit-card">
|
||||
<h3>Unit CCT-310</h3>
|
||||
<p>Studio - $1,100/mo</p>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<!-- Unit 4: Unavailable unit -->
|
||||
<article
|
||||
data-spaces-id="1004"
|
||||
data-spaces-unit="CCT-415"
|
||||
data-spaces-unit-id="5004"
|
||||
data-spaces-unit-floor="4"
|
||||
data-spaces-sort-area="1200"
|
||||
data-spaces-sort-bed="3"
|
||||
data-spaces-sort-bath="2"
|
||||
data-spaces-sort-price="2500"
|
||||
data-spaces-available="false"
|
||||
data-spaces-unavailable="true"
|
||||
data-spaces-soonest=""
|
||||
data-spaces-sort-date=""
|
||||
data-spaces-plan-id="204"
|
||||
data-spaces-sort-plan-name="Dogwood"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100"
|
||||
data-spaces-href="/units/CCT-415"
|
||||
data-spaces-inventory-href="/inventory/CCT-415"
|
||||
data-spaces-specials-content=""
|
||||
>
|
||||
<div class="unit-card">
|
||||
<img src="https://example.com/images/unit-415.jpg" alt="Unit 415" />
|
||||
<h3>Unit CCT-415</h3>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<!-- Unit 5: Minimal attributes - edge case -->
|
||||
<article
|
||||
data-spaces-id="1005"
|
||||
data-spaces-unit="CCT-520"
|
||||
data-spaces-unit-id="5005"
|
||||
data-spaces-sort-price="1300"
|
||||
data-spaces-available="true"
|
||||
data-spaces-unavailable="false"
|
||||
data-spaces-obj="unit"
|
||||
data-spaces-community="Country Club Towers"
|
||||
data-spaces-asset="100"
|
||||
>
|
||||
<div class="unit-card">
|
||||
<h3>Unit CCT-520</h3>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
360
__tests__/scraper/parseUnits.test.js
Normal file
360
__tests__/scraper/parseUnits.test.js
Normal file
@ -0,0 +1,360 @@
|
||||
/**
|
||||
* Tests for parseUnits() - HTML parsing with cheerio
|
||||
*
|
||||
* Covers:
|
||||
* - Extraction of all data-spaces-* attributes
|
||||
* - Image URL extraction from nested img elements
|
||||
* - Container detection (section.spaces__tab-unit)
|
||||
* - Empty / missing container handling
|
||||
* - Deduplication by unit_code
|
||||
* - Logging behavior (info, warn, error)
|
||||
* - Missing attribute handling
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { parseUnits } = require('../../services/scraperService');
|
||||
|
||||
const fixturesDir = path.join(__dirname, 'fixtures');
|
||||
|
||||
// Helper: create a mock logger
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
describe('parseUnits', () => {
|
||||
let sampleHtml;
|
||||
let mockLogger;
|
||||
|
||||
beforeAll(() => {
|
||||
sampleHtml = fs.readFileSync(
|
||||
path.join(fixturesDir, 'sample-listing.html'),
|
||||
'utf-8'
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger = createMockLogger();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1. Container detection
|
||||
// ---------------------------------------------------------------
|
||||
describe('container detection', () => {
|
||||
it('should find section.spaces__tab-unit container in valid HTML', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
// Should not log an error about missing container
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
expect(units.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should return empty array when section container is missing', () => {
|
||||
const html = '<html><body><div>No units here</div></body></html>';
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Container section.spaces__tab-unit not found'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
|
||||
it('should return empty array for empty HTML string', () => {
|
||||
const units = parseUnits('', mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return empty array when container exists but has no articles', () => {
|
||||
const html = '<html><body><section class="spaces__tab-unit"></section></body></html>';
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
expect(units).toEqual([]);
|
||||
// Container found, no error about container
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. Extraction of all data-spaces-* attributes
|
||||
// ---------------------------------------------------------------
|
||||
describe('attribute extraction', () => {
|
||||
it('should extract all expected attributes from a fully populated article', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit1 = units.find(u => u.unit_code === 'CCT-101');
|
||||
|
||||
expect(unit1).toBeDefined();
|
||||
|
||||
// Core identifiers
|
||||
expect(unit1.id).toBe('1001');
|
||||
expect(unit1.unit_code).toBe('CCT-101');
|
||||
expect(unit1.unit_id).toBe('5001');
|
||||
|
||||
// Physical attributes
|
||||
expect(unit1.floor).toBe('1');
|
||||
expect(unit1.area).toBe('750');
|
||||
expect(unit1.bed_count).toBe('1');
|
||||
expect(unit1.bath_count).toBe('1');
|
||||
|
||||
// Pricing
|
||||
expect(unit1.price).toBe('1450');
|
||||
|
||||
// Availability
|
||||
expect(unit1.available).toBe('true');
|
||||
expect(unit1.unavailable).toBe('false');
|
||||
expect(unit1.soonest).toBe('2026-03-01');
|
||||
expect(unit1.date_available).toBe('20260301');
|
||||
|
||||
// Plan information
|
||||
expect(unit1.plan_id).toBe('201');
|
||||
expect(unit1.plan_name).toBe('Alpine');
|
||||
|
||||
// Property information
|
||||
expect(unit1.obj_type).toBe('unit');
|
||||
expect(unit1.community).toBe('Country Club Towers');
|
||||
expect(unit1.asset).toBe('100');
|
||||
|
||||
// URLs
|
||||
expect(unit1.href).toBe('/units/CCT-101');
|
||||
expect(unit1.inventory_href).toBe('/inventory/CCT-101');
|
||||
|
||||
// Specials
|
||||
expect(unit1.specials_content).toBe('First month free!');
|
||||
});
|
||||
|
||||
it('should extract correct count of units from the fixture', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
expect(units).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('should return raw string values without type conversion', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit = units.find(u => u.unit_code === 'CCT-101');
|
||||
|
||||
// All values should be strings (raw extraction, no conversion)
|
||||
expect(typeof unit.id).toBe('string');
|
||||
expect(typeof unit.price).toBe('string');
|
||||
expect(typeof unit.floor).toBe('string');
|
||||
expect(typeof unit.available).toBe('string');
|
||||
expect(typeof unit.bed_count).toBe('string');
|
||||
expect(typeof unit.bath_count).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Image URL extraction
|
||||
// ---------------------------------------------------------------
|
||||
describe('image URL extraction', () => {
|
||||
it('should extract image_url from img src attribute', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit1 = units.find(u => u.unit_code === 'CCT-101');
|
||||
|
||||
expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
|
||||
});
|
||||
|
||||
it('should fall back to data-src when src is not present', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit2 = units.find(u => u.unit_code === 'CCT-205');
|
||||
|
||||
expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
|
||||
});
|
||||
|
||||
it('should not set image_url when no img element exists', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit3 = units.find(u => u.unit_code === 'CCT-310');
|
||||
|
||||
expect(unit3.image_url).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Missing attributes
|
||||
// ---------------------------------------------------------------
|
||||
describe('missing attributes', () => {
|
||||
it('should return undefined for attributes not present on the article', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
// Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
|
||||
// soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
|
||||
const unit5 = units.find(u => u.unit_code === 'CCT-520');
|
||||
|
||||
expect(unit5).toBeDefined();
|
||||
expect(unit5.floor).toBeUndefined();
|
||||
expect(unit5.area).toBeUndefined();
|
||||
expect(unit5.bed_count).toBeUndefined();
|
||||
expect(unit5.bath_count).toBeUndefined();
|
||||
expect(unit5.soonest).toBeUndefined();
|
||||
expect(unit5.date_available).toBeUndefined();
|
||||
expect(unit5.plan_id).toBeUndefined();
|
||||
expect(unit5.plan_name).toBeUndefined();
|
||||
expect(unit5.href).toBeUndefined();
|
||||
expect(unit5.inventory_href).toBeUndefined();
|
||||
expect(unit5.specials_content).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should still extract the attributes that are present on a sparse article', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit5 = units.find(u => u.unit_code === 'CCT-520');
|
||||
|
||||
expect(unit5.id).toBe('1005');
|
||||
expect(unit5.unit_code).toBe('CCT-520');
|
||||
expect(unit5.unit_id).toBe('5005');
|
||||
expect(unit5.price).toBe('1300');
|
||||
expect(unit5.available).toBe('true');
|
||||
expect(unit5.obj_type).toBe('unit');
|
||||
expect(unit5.community).toBe('Country Club Towers');
|
||||
expect(unit5.asset).toBe('100');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Deduplication
|
||||
// ---------------------------------------------------------------
|
||||
describe('deduplication', () => {
|
||||
it('should deduplicate units with the same unit_code', () => {
|
||||
// Build HTML with duplicate articles
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
const codes = units.map(u => u.unit_code);
|
||||
|
||||
// Should have 2 unique unit codes
|
||||
expect(units).toHaveLength(2);
|
||||
expect(new Set(codes).size).toBe(codes.length);
|
||||
});
|
||||
|
||||
it('should keep the first occurrence when deduplicating', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
// First occurrence (id=1, price=1000) should be kept
|
||||
expect(units[0].id).toBe('1');
|
||||
expect(units[0].price).toBe('1000');
|
||||
});
|
||||
|
||||
it('should log a warning when duplicates are removed', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-unit="DUPE-1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="DUPE-1" data-spaces-sort-price="1100"></article>
|
||||
<article data-spaces-id="3" data-spaces-unit="UNIQUE-1" data-spaces-sort-price="1200"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
parseUnits(html, mockLogger);
|
||||
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(
|
||||
'Duplicate units removed',
|
||||
expect.objectContaining({
|
||||
original: 3,
|
||||
deduplicated: 2
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should not log a warning when there are no duplicates', () => {
|
||||
parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
// The fixture has no duplicates, so no warn about duplicates
|
||||
const warnCalls = mockLogger.warn.mock.calls;
|
||||
const duplicateWarns = warnCalls.filter(
|
||||
call => call[0] === 'Duplicate units removed'
|
||||
);
|
||||
expect(duplicateWarns).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should filter out articles with no unit_code during deduplication', () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<section class="spaces__tab-unit">
|
||||
<article data-spaces-id="1" data-spaces-sort-price="1000"></article>
|
||||
<article data-spaces-id="2" data-spaces-unit="VALID-1" data-spaces-sort-price="1100"></article>
|
||||
</section>
|
||||
</body></html>
|
||||
`;
|
||||
|
||||
const units = parseUnits(html, mockLogger);
|
||||
|
||||
// Article without unit_code should be filtered out
|
||||
expect(units).toHaveLength(1);
|
||||
expect(units[0].unit_code).toBe('VALID-1');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. Logging behavior
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log the count of parsed units', () => {
|
||||
parseUnits(sampleHtml, mockLogger);
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'Units parsed',
|
||||
expect.objectContaining({ count: 5 })
|
||||
);
|
||||
});
|
||||
|
||||
it('should log error when container is not found', () => {
|
||||
const html = '<html><body><p>Nothing here</p></body></html>';
|
||||
parseUnits(html, mockLogger);
|
||||
|
||||
expect(mockLogger.error).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Container section.spaces__tab-unit not found'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Multiple units with varied data
|
||||
// ---------------------------------------------------------------
|
||||
describe('varied unit data', () => {
|
||||
it('should handle unavailable units correctly', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit4 = units.find(u => u.unit_code === 'CCT-415');
|
||||
|
||||
expect(unit4.available).toBe('false');
|
||||
expect(unit4.unavailable).toBe('true');
|
||||
expect(unit4.soonest).toBe('');
|
||||
expect(unit4.date_available).toBe('');
|
||||
});
|
||||
|
||||
it('should handle half bath counts', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit2 = units.find(u => u.unit_code === 'CCT-205');
|
||||
|
||||
expect(unit2.bath_count).toBe('1.5');
|
||||
});
|
||||
|
||||
it('should handle studio units (bed_count = 0)', () => {
|
||||
const units = parseUnits(sampleHtml, mockLogger);
|
||||
const unit3 = units.find(u => u.unit_code === 'CCT-310');
|
||||
|
||||
expect(unit3.bed_count).toBe('0');
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const config = require('../config/scraper');
|
||||
|
||||
/**
|
||||
@ -106,8 +107,100 @@ async function fetchPage(url, logger) {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse unit data from HTML using cheerio
|
||||
* @param {string} html - HTML content
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
|
||||
*/
|
||||
function parseUnits(html, logger) {
|
||||
const $ = cheerio.load(html);
|
||||
const units = [];
|
||||
|
||||
// Find the main container
|
||||
const container = $('section.spaces__tab-unit');
|
||||
|
||||
if (container.length === 0) {
|
||||
logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
|
||||
return [];
|
||||
}
|
||||
|
||||
// Extract each article element
|
||||
container.find('article').each((index, article) => {
|
||||
const $article = $(article);
|
||||
|
||||
const unit = {
|
||||
// Core identifiers
|
||||
id: $article.attr('data-spaces-id'),
|
||||
unit_code: $article.attr('data-spaces-unit'),
|
||||
unit_id: $article.attr('data-spaces-unit-id'),
|
||||
|
||||
// Physical attributes
|
||||
floor: $article.attr('data-spaces-unit-floor'),
|
||||
area: $article.attr('data-spaces-sort-area'),
|
||||
bed_count: $article.attr('data-spaces-sort-bed'),
|
||||
bath_count: $article.attr('data-spaces-sort-bath'),
|
||||
|
||||
// Pricing
|
||||
price: $article.attr('data-spaces-sort-price'),
|
||||
|
||||
// Availability
|
||||
available: $article.attr('data-spaces-available'),
|
||||
unavailable: $article.attr('data-spaces-unavailable'),
|
||||
soonest: $article.attr('data-spaces-soonest'),
|
||||
date_available: $article.attr('data-spaces-sort-date'),
|
||||
|
||||
// Plan information
|
||||
plan_id: $article.attr('data-spaces-plan-id'),
|
||||
plan_name: $article.attr('data-spaces-sort-plan-name'),
|
||||
|
||||
// Property information
|
||||
obj_type: $article.attr('data-spaces-obj'),
|
||||
community: $article.attr('data-spaces-community'),
|
||||
asset: $article.attr('data-spaces-asset'),
|
||||
|
||||
// URLs
|
||||
href: $article.attr('data-spaces-href'),
|
||||
inventory_href: $article.attr('data-spaces-inventory-href'),
|
||||
|
||||
// Specials
|
||||
specials_content: $article.attr('data-spaces-specials-content')
|
||||
};
|
||||
|
||||
// Extract image URL from nested element if present
|
||||
const imgElement = $article.find('img').first();
|
||||
if (imgElement.length > 0) {
|
||||
unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
|
||||
}
|
||||
|
||||
units.push(unit);
|
||||
});
|
||||
|
||||
logger.info('Units parsed', { count: units.length });
|
||||
|
||||
// Deduplicate by unit_code (in case of duplicate articles)
|
||||
const seen = new Set();
|
||||
const deduplicated = units.filter(unit => {
|
||||
if (!unit.unit_code || seen.has(unit.unit_code)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(unit.unit_code);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (deduplicated.length < units.length) {
|
||||
logger.warn('Duplicate units removed', {
|
||||
original: units.length,
|
||||
deduplicated: deduplicated.length
|
||||
});
|
||||
}
|
||||
|
||||
return deduplicated;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchPage,
|
||||
parseUnits,
|
||||
// Export for testing
|
||||
isRetryableError,
|
||||
sleep
|
||||
|
||||
Reference in New Issue
Block a user