Implement parseUnits() HTML parsing with cheerio
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Run Linting (pull_request) Successful in 9m37s
CI/CD Pipeline - Apartment API / Run Tests (pull_request) Successful in 9m43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped

- Extract apartment unit data from section.spaces__tab-unit container
- Parse all data-spaces-* attributes from article elements
- Deduplicate units by unit_code with warning logging
- Handle missing container gracefully (returns empty array)
- Add comprehensive test suite with HTML fixture (22 tests)
This commit is contained in:
2026-02-05 16:49:41 -07:00
parent c6beaf8333
commit 39b513cbba
3 changed files with 597 additions and 0 deletions

View File

@ -6,6 +6,7 @@
*/
const axios = require('axios');
const cheerio = require('cheerio');
const config = require('../config/scraper');
/**
@ -106,8 +107,100 @@ async function fetchPage(url, logger) {
throw lastError;
}
/**
* Parse unit data from HTML using cheerio
* @param {string} html - HTML content
* @param {Object} logger - Logger instance
* @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
*/
function parseUnits(html, logger) {
const $ = cheerio.load(html);
const units = [];
// Find the main container
const container = $('section.spaces__tab-unit');
if (container.length === 0) {
logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
return [];
}
// Extract each article element
container.find('article').each((index, article) => {
const $article = $(article);
const unit = {
// Core identifiers
id: $article.attr('data-spaces-id'),
unit_code: $article.attr('data-spaces-unit'),
unit_id: $article.attr('data-spaces-unit-id'),
// Physical attributes
floor: $article.attr('data-spaces-unit-floor'),
area: $article.attr('data-spaces-sort-area'),
bed_count: $article.attr('data-spaces-sort-bed'),
bath_count: $article.attr('data-spaces-sort-bath'),
// Pricing
price: $article.attr('data-spaces-sort-price'),
// Availability
available: $article.attr('data-spaces-available'),
unavailable: $article.attr('data-spaces-unavailable'),
soonest: $article.attr('data-spaces-soonest'),
date_available: $article.attr('data-spaces-sort-date'),
// Plan information
plan_id: $article.attr('data-spaces-plan-id'),
plan_name: $article.attr('data-spaces-sort-plan-name'),
// Property information
obj_type: $article.attr('data-spaces-obj'),
community: $article.attr('data-spaces-community'),
asset: $article.attr('data-spaces-asset'),
// URLs
href: $article.attr('data-spaces-href'),
inventory_href: $article.attr('data-spaces-inventory-href'),
// Specials
specials_content: $article.attr('data-spaces-specials-content')
};
// Extract image URL from nested element if present
const imgElement = $article.find('img').first();
if (imgElement.length > 0) {
unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
}
units.push(unit);
});
logger.info('Units parsed', { count: units.length });
// Deduplicate by unit_code (in case of duplicate articles)
const seen = new Set();
const deduplicated = units.filter(unit => {
if (!unit.unit_code || seen.has(unit.unit_code)) {
return false;
}
seen.add(unit.unit_code);
return true;
});
if (deduplicated.length < units.length) {
logger.warn('Duplicate units removed', {
original: units.length,
deduplicated: deduplicated.length
});
}
return deduplicated;
}
module.exports = {
fetchPage,
parseUnits,
// Export for testing
isRetryableError,
sleep