Implement parseUnits() HTML parsing with cheerio
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Run Linting (pull_request) Successful in 9m37s
CI/CD Pipeline - Apartment API / Run Tests (pull_request) Successful in 9m43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Run Linting (pull_request) Successful in 9m37s
CI/CD Pipeline - Apartment API / Run Tests (pull_request) Successful in 9m43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
- Extract apartment unit data from section.spaces__tab-unit container - Parse all data-spaces-* attributes from article elements - Deduplicate units by unit_code with warning logging - Handle missing container gracefully (returns empty array) - Add comprehensive test suite with HTML fixture (22 tests)
This commit is contained in:
@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const config = require('../config/scraper');
|
||||
|
||||
/**
|
||||
@ -106,8 +107,100 @@ async function fetchPage(url, logger) {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse unit data from HTML using cheerio
|
||||
* @param {string} html - HTML content
|
||||
* @param {Object} logger - Logger instance
|
||||
* @returns {Array<Object>} Array of raw unit objects (string values, no type conversion)
|
||||
*/
|
||||
function parseUnits(html, logger) {
|
||||
const $ = cheerio.load(html);
|
||||
const units = [];
|
||||
|
||||
// Find the main container
|
||||
const container = $('section.spaces__tab-unit');
|
||||
|
||||
if (container.length === 0) {
|
||||
logger.error('Container section.spaces__tab-unit not found - possible structure change', {});
|
||||
return [];
|
||||
}
|
||||
|
||||
// Extract each article element
|
||||
container.find('article').each((index, article) => {
|
||||
const $article = $(article);
|
||||
|
||||
const unit = {
|
||||
// Core identifiers
|
||||
id: $article.attr('data-spaces-id'),
|
||||
unit_code: $article.attr('data-spaces-unit'),
|
||||
unit_id: $article.attr('data-spaces-unit-id'),
|
||||
|
||||
// Physical attributes
|
||||
floor: $article.attr('data-spaces-unit-floor'),
|
||||
area: $article.attr('data-spaces-sort-area'),
|
||||
bed_count: $article.attr('data-spaces-sort-bed'),
|
||||
bath_count: $article.attr('data-spaces-sort-bath'),
|
||||
|
||||
// Pricing
|
||||
price: $article.attr('data-spaces-sort-price'),
|
||||
|
||||
// Availability
|
||||
available: $article.attr('data-spaces-available'),
|
||||
unavailable: $article.attr('data-spaces-unavailable'),
|
||||
soonest: $article.attr('data-spaces-soonest'),
|
||||
date_available: $article.attr('data-spaces-sort-date'),
|
||||
|
||||
// Plan information
|
||||
plan_id: $article.attr('data-spaces-plan-id'),
|
||||
plan_name: $article.attr('data-spaces-sort-plan-name'),
|
||||
|
||||
// Property information
|
||||
obj_type: $article.attr('data-spaces-obj'),
|
||||
community: $article.attr('data-spaces-community'),
|
||||
asset: $article.attr('data-spaces-asset'),
|
||||
|
||||
// URLs
|
||||
href: $article.attr('data-spaces-href'),
|
||||
inventory_href: $article.attr('data-spaces-inventory-href'),
|
||||
|
||||
// Specials
|
||||
specials_content: $article.attr('data-spaces-specials-content')
|
||||
};
|
||||
|
||||
// Extract image URL from nested element if present
|
||||
const imgElement = $article.find('img').first();
|
||||
if (imgElement.length > 0) {
|
||||
unit.image_url = imgElement.attr('src') || imgElement.attr('data-src');
|
||||
}
|
||||
|
||||
units.push(unit);
|
||||
});
|
||||
|
||||
logger.info('Units parsed', { count: units.length });
|
||||
|
||||
// Deduplicate by unit_code (in case of duplicate articles)
|
||||
const seen = new Set();
|
||||
const deduplicated = units.filter(unit => {
|
||||
if (!unit.unit_code || seen.has(unit.unit_code)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(unit.unit_code);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (deduplicated.length < units.length) {
|
||||
logger.warn('Duplicate units removed', {
|
||||
original: units.length,
|
||||
deduplicated: deduplicated.length
|
||||
});
|
||||
}
|
||||
|
||||
return deduplicated;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchPage,
|
||||
parseUnits,
|
||||
// Export for testing
|
||||
isRetryableError,
|
||||
sleep
|
||||
|
||||
Reference in New Issue
Block a user