stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

SCRAPE-24: Create HTML fixture files for scraper tests #28

Merged
stephen merged 2 commits from scraper/fixtures into dev 2026-02-07 10:58:14 -07:00
Conversation 1 Commits 2 Files Changed 4 +494 -206
4 changed files with 494 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fcfd35aa3f - Show all commits

201
__tests__/scraper/parseUnits.test.js
View File

@ -87,55 +87,55 @@ describe('parseUnits', () => {
describe('attribute extraction', () => { describe('attribute extraction', () => {
it('should extract all expected attributes from a fully populated article', () => { it('should extract all expected attributes from a fully populated article', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'CCT-101'); const unit1 = units.find(u => u.unit_code === 'W2707');
expect(unit1).toBeDefined(); expect(unit1).toBeDefined();
// Core identifiers // Core identifiers
expect(unit1.id).toBe('1001'); expect(unit1.id).toBe('154842');
expect(unit1.unit_code).toBe('CCT-101'); expect(unit1.unit_code).toBe('W2707');
expect(unit1.unit_id).toBe('5001'); expect(unit1.unit_id).toBe('154842');
// Physical attributes // Physical attributes
expect(unit1.floor).toBe('1'); expect(unit1.floor).toBe('2755');
expect(unit1.area).toBe('750'); expect(unit1.area).toBe('1210');
expect(unit1.bed_count).toBe('1'); expect(unit1.bed_count).toBe('2');
expect(unit1.bath_count).toBe('1'); expect(unit1.bath_count).toBe('2');
// Pricing // Pricing
expect(unit1.price).toBe('1450'); expect(unit1.price).toBe('5230');
// Availability // Availability
expect(unit1.available).toBe('true'); expect(unit1.available).toBe('true');
expect(unit1.unavailable).toBe('false'); expect(unit1.unavailable).toBe('false');
expect(unit1.soonest).toBe('2026-03-01'); expect(unit1.soonest).toBe('2025-10-11');
expect(unit1.date_available).toBe('20260301'); expect(unit1.date_available).toBe('1760140800');
// Plan information // Plan information
expect(unit1.plan_id).toBe('201'); expect(unit1.plan_id).toBe('10270');
expect(unit1.plan_name).toBe('Alpine'); expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
// Property information // Property information
expect(unit1.obj_type).toBe('unit'); expect(unit1.obj_type).toBe('unit');
expect(unit1.community).toBe('Country Club Towers'); expect(unit1.community).toBe('Country Club Towers');
expect(unit1.asset).toBe('100'); expect(unit1.asset).toBe('420');
// URLs // URLs
expect(unit1.href).toBe('/units/CCT-101'); expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
expect(unit1.inventory_href).toBe('/inventory/CCT-101'); expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
// Specials // Specials
expect(unit1.specials_content).toBe('First month free!'); expect(unit1.specials_content).toBe('');
}); });
it('should extract correct count of units from the fixture', () => { it('should extract correct count of units from the fixture', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
expect(units).toHaveLength(5); expect(units).toHaveLength(10);
}); });
it('should return raw string values without type conversion', () => { it('should return raw string values without type conversion', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'CCT-101'); const unit = units.find(u => u.unit_code === 'W2707');
// All values should be strings (raw extraction, no conversion) // All values should be strings (raw extraction, no conversion)
expect(typeof unit.id).toBe('string'); expect(typeof unit.id).toBe('string');
@ -152,24 +152,48 @@ describe('parseUnits', () => {
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('image URL extraction', () => { describe('image URL extraction', () => {
it('should extract image_url from img src attribute', () => { it('should extract image_url from img src attribute', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
const unit1 = units.find(u => u.unit_code === 'CCT-101'); <html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
<img src="https://example.com/images/unit-1.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg'); expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
}); });
it('should fall back to data-src when src is not present', () => { it('should fall back to data-src when src is not present', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
const unit2 = units.find(u => u.unit_code === 'CCT-205'); <html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
<img data-src="https://example.com/images/unit-2.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg'); expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
}); });
it('should not set image_url when no img element exists', () => { it('should not set image_url when no img element exists', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
const unit3 = units.find(u => u.unit_code === 'CCT-310'); <html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
<div>No image here</div>
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit3.image_url).toBeUndefined(); expect(units[0].image_url).toBeUndefined();
}); });
}); });
@ -178,37 +202,67 @@ describe('parseUnits', () => {
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('missing attributes', () => { describe('missing attributes', () => {
it('should return undefined for attributes not present on the article', () => { it('should return undefined for attributes not present on the article', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
// Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count, <html><body>
// soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content <section class="spaces__tab-unit">
const unit5 = units.find(u => u.unit_code === 'CCT-520'); <article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit5).toBeDefined(); expect(unit).toBeDefined();
expect(unit5.floor).toBeUndefined(); expect(unit.floor).toBeUndefined();
expect(unit5.area).toBeUndefined(); expect(unit.area).toBeUndefined();
expect(unit5.bed_count).toBeUndefined(); expect(unit.bed_count).toBeUndefined();
expect(unit5.bath_count).toBeUndefined(); expect(unit.bath_count).toBeUndefined();
expect(unit5.soonest).toBeUndefined(); expect(unit.soonest).toBeUndefined();
expect(unit5.date_available).toBeUndefined(); expect(unit.date_available).toBeUndefined();
expect(unit5.plan_id).toBeUndefined(); expect(unit.plan_id).toBeUndefined();
expect(unit5.plan_name).toBeUndefined(); expect(unit.plan_name).toBeUndefined();
expect(unit5.href).toBeUndefined(); expect(unit.href).toBeUndefined();
expect(unit5.inventory_href).toBeUndefined(); expect(unit.inventory_href).toBeUndefined();
expect(unit5.specials_content).toBeUndefined(); expect(unit.specials_content).toBeUndefined();
}); });
it('should still extract the attributes that are present on a sparse article', () => { it('should still extract the attributes that are present on a sparse article', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
const unit5 = units.find(u => u.unit_code === 'CCT-520'); <html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit5.id).toBe('1005'); expect(unit.id).toBe('1005');
expect(unit5.unit_code).toBe('CCT-520'); expect(unit.unit_code).toBe('SPARSE-1');
expect(unit5.unit_id).toBe('5005'); expect(unit.unit_id).toBe('5005');
expect(unit5.price).toBe('1300'); expect(unit.price).toBe('1300');
expect(unit5.available).toBe('true'); expect(unit.available).toBe('true');
expect(unit5.obj_type).toBe('unit'); expect(unit.obj_type).toBe('unit');
expect(unit5.community).toBe('Country Club Towers'); expect(unit.community).toBe('Country Club Towers');
expect(unit5.asset).toBe('100'); expect(unit.asset).toBe('100');
}); });
}); });
@ -313,7 +367,7 @@ describe('parseUnits', () => {
expect(mockLogger.info).toHaveBeenCalledWith( expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed', 'Units parsed',
expect.objectContaining({ count: 5 }) expect.objectContaining({ count: 10 })
); );
}); });
@ -334,27 +388,44 @@ describe('parseUnits', () => {
// --------------------------------------------------------------- // ---------------------------------------------------------------
describe('varied unit data', () => { describe('varied unit data', () => {
it('should handle unavailable units correctly', () => { it('should handle unavailable units correctly', () => {
const units = parseUnits(sampleHtml, mockLogger); const html = `
const unit4 = units.find(u => u.unit_code === 'CCT-415'); <html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="4001"
data-spaces-unit="UNAVAIL-1"
data-spaces-sort-price="2000"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date="">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit4.available).toBe('false'); expect(unit.available).toBe('false');
expect(unit4.unavailable).toBe('true'); expect(unit.unavailable).toBe('true');
expect(unit4.soonest).toBe(''); expect(unit.soonest).toBe('');
expect(unit4.date_available).toBe(''); expect(unit.date_available).toBe('');
}); });
it('should handle half bath counts', () => { it('should handle half bath counts', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit2 = units.find(u => u.unit_code === 'CCT-205'); // E3205 is the penthouse with 2.5 baths
const unit = units.find(u => u.unit_code === 'E3205');
expect(unit2.bath_count).toBe('1.5'); expect(unit.bath_count).toBe('2.5');
}); });
it('should handle studio units (bed_count = 0)', () => { it('should handle studio units (bed_count = 0)', () => {
const units = parseUnits(sampleHtml, mockLogger); const units = parseUnits(sampleHtml, mockLogger);
const unit3 = units.find(u => u.unit_code === 'CCT-310'); // W2603 is the studio with bed_count 0
const unit = units.find(u => u.unit_code === 'W2603');
expect(unit3.bed_count).toBe('0'); expect(unit.bed_count).toBe('0');
}); });
}); });
}); });