Compare commits

..

3 Commits

Author SHA1 Message Date
fcfd35aa3f Update parseUnits tests to match real fixture data
All checks were successful
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 43s
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
The fixture file was updated with real website data (10 units) but
the tests still referenced old synthetic data (5 units with CCT-*
codes). Updated tests to use real unit codes from the fixture (W2707,
E3205, W2603) and moved edge case tests (image extraction, missing
attributes, unavailable units) to use inline HTML for isolation.
2026-02-07 10:56:37 -07:00
724115f2b3 Add HTML fixture files for scraper unit tests
Create three HTML fixture files in __tests__/scraper/fixtures/ to support
deterministic testing of the scraper's HTML parsing logic:

- sample-listing.html: Contains 10 real unit articles extracted from the
  live listings page, covering studios, 1BR, and 2BR floor plans with
  varied pricing and availability dates.

- sample-listing-empty.html: Minimal page structure with an empty units
  section, for testing graceful handling of pages with no listings.

- sample-listing-call.html: Contains units with "Call for pricing" instead
  of numeric rent values, for testing the parser's handling of non-numeric
  price fields.
2026-02-07 10:53:04 -07:00
6f1651436d Fix webhook notification failure when CI tests produce large output
Truncate lint/test output to 10000 chars at the source (CI job) before
writing to GITHUB_OUTPUT, instead of in the downstream notify job.
Previously the full output was passed as an env var between jobs, which
could exceed Linux's ARG_MAX limit and prevent bash from launching.
2026-02-07 10:51:28 -07:00
2 changed files with 145 additions and 70 deletions

View File

@ -44,8 +44,10 @@ jobs:
set +e
OUTPUT=$(npm run lint 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "lint_output<<EOF" >> $GITHUB_OUTPUT
echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
@ -56,8 +58,10 @@ jobs:
set +e
OUTPUT=$(npm test -- --runInBand 2>&1)
EXIT_CODE=$?
# Truncate before writing to GITHUB_OUTPUT to prevent
# "argument list too long" in downstream jobs
echo "test_output<<EOF" >> $GITHUB_OUTPUT
echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE
env:
@ -98,9 +102,9 @@ jobs:
OVERALL_STATUS="failure"
fi
# Truncate outputs if too long (max 10000 chars each)
LINT_OUTPUT="${RAW_LINT_OUTPUT:0:10000}"
TEST_OUTPUT="${RAW_TEST_OUTPUT:0:10000}"
# Outputs are already truncated at the source (ci job)
LINT_OUTPUT="$RAW_LINT_OUTPUT"
TEST_OUTPUT="$RAW_TEST_OUTPUT"
# Build JSON payload
PAYLOAD=$(jq -n \

View File

@ -87,55 +87,55 @@ describe('parseUnits', () => {
describe('attribute extraction', () => {
it('should extract all expected attributes from a fully populated article', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'CCT-101');
const unit1 = units.find(u => u.unit_code === 'W2707');
expect(unit1).toBeDefined();
// Core identifiers
expect(unit1.id).toBe('1001');
expect(unit1.unit_code).toBe('CCT-101');
expect(unit1.unit_id).toBe('5001');
expect(unit1.id).toBe('154842');
expect(unit1.unit_code).toBe('W2707');
expect(unit1.unit_id).toBe('154842');
// Physical attributes
expect(unit1.floor).toBe('1');
expect(unit1.area).toBe('750');
expect(unit1.bed_count).toBe('1');
expect(unit1.bath_count).toBe('1');
expect(unit1.floor).toBe('2755');
expect(unit1.area).toBe('1210');
expect(unit1.bed_count).toBe('2');
expect(unit1.bath_count).toBe('2');
// Pricing
expect(unit1.price).toBe('1450');
expect(unit1.price).toBe('5230');
// Availability
expect(unit1.available).toBe('true');
expect(unit1.unavailable).toBe('false');
expect(unit1.soonest).toBe('2026-03-01');
expect(unit1.date_available).toBe('20260301');
expect(unit1.soonest).toBe('2025-10-11');
expect(unit1.date_available).toBe('1760140800');
// Plan information
expect(unit1.plan_id).toBe('201');
expect(unit1.plan_name).toBe('Alpine');
expect(unit1.plan_id).toBe('10270');
expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
// Property information
expect(unit1.obj_type).toBe('unit');
expect(unit1.community).toBe('Country Club Towers');
expect(unit1.asset).toBe('100');
expect(unit1.asset).toBe('420');
// URLs
expect(unit1.href).toBe('/units/CCT-101');
expect(unit1.inventory_href).toBe('/inventory/CCT-101');
expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
// Specials
expect(unit1.specials_content).toBe('First month free!');
expect(unit1.specials_content).toBe('');
});
it('should extract correct count of units from the fixture', () => {
const units = parseUnits(sampleHtml, mockLogger);
expect(units).toHaveLength(5);
expect(units).toHaveLength(10);
});
it('should return raw string values without type conversion', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit = units.find(u => u.unit_code === 'CCT-101');
const unit = units.find(u => u.unit_code === 'W2707');
// All values should be strings (raw extraction, no conversion)
expect(typeof unit.id).toBe('string');
@ -152,24 +152,48 @@ describe('parseUnits', () => {
// ---------------------------------------------------------------
describe('image URL extraction', () => {
it('should extract image_url from img src attribute', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit1 = units.find(u => u.unit_code === 'CCT-101');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
<img src="https://example.com/images/unit-1.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
});
it('should fall back to data-src when src is not present', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit2 = units.find(u => u.unit_code === 'CCT-205');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
<img data-src="https://example.com/images/unit-2.jpg" />
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
});
it('should not set image_url when no img element exists', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit3 = units.find(u => u.unit_code === 'CCT-310');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
<div>No image here</div>
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
expect(unit3.image_url).toBeUndefined();
expect(units[0].image_url).toBeUndefined();
});
});
@ -178,37 +202,67 @@ describe('parseUnits', () => {
// ---------------------------------------------------------------
describe('missing attributes', () => {
it('should return undefined for attributes not present on the article', () => {
const units = parseUnits(sampleHtml, mockLogger);
// Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
// soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
const unit5 = units.find(u => u.unit_code === 'CCT-520');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit5).toBeDefined();
expect(unit5.floor).toBeUndefined();
expect(unit5.area).toBeUndefined();
expect(unit5.bed_count).toBeUndefined();
expect(unit5.bath_count).toBeUndefined();
expect(unit5.soonest).toBeUndefined();
expect(unit5.date_available).toBeUndefined();
expect(unit5.plan_id).toBeUndefined();
expect(unit5.plan_name).toBeUndefined();
expect(unit5.href).toBeUndefined();
expect(unit5.inventory_href).toBeUndefined();
expect(unit5.specials_content).toBeUndefined();
expect(unit).toBeDefined();
expect(unit.floor).toBeUndefined();
expect(unit.area).toBeUndefined();
expect(unit.bed_count).toBeUndefined();
expect(unit.bath_count).toBeUndefined();
expect(unit.soonest).toBeUndefined();
expect(unit.date_available).toBeUndefined();
expect(unit.plan_id).toBeUndefined();
expect(unit.plan_name).toBeUndefined();
expect(unit.href).toBeUndefined();
expect(unit.inventory_href).toBeUndefined();
expect(unit.specials_content).toBeUndefined();
});
it('should still extract the attributes that are present on a sparse article', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit5 = units.find(u => u.unit_code === 'CCT-520');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="1005"
data-spaces-unit="SPARSE-1"
data-spaces-unit-id="5005"
data-spaces-sort-price="1300"
data-spaces-available="true"
data-spaces-obj="unit"
data-spaces-community="Country Club Towers"
data-spaces-asset="100">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit5.id).toBe('1005');
expect(unit5.unit_code).toBe('CCT-520');
expect(unit5.unit_id).toBe('5005');
expect(unit5.price).toBe('1300');
expect(unit5.available).toBe('true');
expect(unit5.obj_type).toBe('unit');
expect(unit5.community).toBe('Country Club Towers');
expect(unit5.asset).toBe('100');
expect(unit.id).toBe('1005');
expect(unit.unit_code).toBe('SPARSE-1');
expect(unit.unit_id).toBe('5005');
expect(unit.price).toBe('1300');
expect(unit.available).toBe('true');
expect(unit.obj_type).toBe('unit');
expect(unit.community).toBe('Country Club Towers');
expect(unit.asset).toBe('100');
});
});
@ -313,7 +367,7 @@ describe('parseUnits', () => {
expect(mockLogger.info).toHaveBeenCalledWith(
'Units parsed',
expect.objectContaining({ count: 5 })
expect.objectContaining({ count: 10 })
);
});
@ -334,27 +388,44 @@ describe('parseUnits', () => {
// ---------------------------------------------------------------
describe('varied unit data', () => {
it('should handle unavailable units correctly', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit4 = units.find(u => u.unit_code === 'CCT-415');
const html = `
<html><body>
<section class="spaces__tab-unit">
<article
data-spaces-id="4001"
data-spaces-unit="UNAVAIL-1"
data-spaces-sort-price="2000"
data-spaces-available="false"
data-spaces-unavailable="true"
data-spaces-soonest=""
data-spaces-sort-date="">
</article>
</section>
</body></html>
`;
const units = parseUnits(html, mockLogger);
const unit = units[0];
expect(unit4.available).toBe('false');
expect(unit4.unavailable).toBe('true');
expect(unit4.soonest).toBe('');
expect(unit4.date_available).toBe('');
expect(unit.available).toBe('false');
expect(unit.unavailable).toBe('true');
expect(unit.soonest).toBe('');
expect(unit.date_available).toBe('');
});
it('should handle half bath counts', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit2 = units.find(u => u.unit_code === 'CCT-205');
// E3205 is the penthouse with 2.5 baths
const unit = units.find(u => u.unit_code === 'E3205');
expect(unit2.bath_count).toBe('1.5');
expect(unit.bath_count).toBe('2.5');
});
it('should handle studio units (bed_count = 0)', () => {
const units = parseUnits(sampleHtml, mockLogger);
const unit3 = units.find(u => u.unit_code === 'CCT-310');
// W2603 is the studio with bed_count 0
const unit = units.find(u => u.unit_code === 'W2603');
expect(unit3.bed_count).toBe('0');
expect(unit.bed_count).toBe('0');
});
});
});