SCRAPE-9: Implement updateDailySummary() aggregation (#14)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
561
__tests__/scraper/updateDailySummary.test.js
Normal file
561
__tests__/scraper/updateDailySummary.test.js
Normal file
@ -0,0 +1,561 @@
|
||||
/**
|
||||
* Tests for updateDailySummary() daily summary aggregation
|
||||
*
|
||||
* Covers:
|
||||
* - Upsert by date key (YYYY-MM-DD)
|
||||
* - Summary includes new_units, rented_units, stale_units arrays
|
||||
* - Count fields are calculated correctly
|
||||
* - net_change = newUnits.length - rentedUnits.length
|
||||
* - total_available_today is set correctly
|
||||
* - total_available_yesterday fetched from previous day
|
||||
* - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
|
||||
* - turnover_rate is 0 when yesterdayTotal is 0
|
||||
* - Error handling when updateOne fails
|
||||
* - Logging of summary update details
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require updateDailySummary after implementation
|
||||
let updateDailySummary;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ updateDailySummary } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('updateDailySummary', () => {
|
||||
// ---------------------------------------------------------------
|
||||
// 1. Upsert by date key
|
||||
// ---------------------------------------------------------------
|
||||
describe('upsert by date key', () => {
|
||||
it('should insert a new summary when no document exists for the date', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 2,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
const result = await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
// Should have upserted (created new)
|
||||
expect(result.upsertedCount).toBe(1);
|
||||
|
||||
// Verify the document in the database
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc).not.toBeNull();
|
||||
expect(doc.date).toBe('2026-02-05');
|
||||
});
|
||||
|
||||
it('should update an existing summary for the same date (idempotent upsert)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert initial summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-05',
|
||||
new_units: ['A101'],
|
||||
rented_units: [],
|
||||
total_available_today: 10
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 1,
|
||||
totalAvailable: 12
|
||||
};
|
||||
|
||||
const result = await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
// Should have matched and modified (not upserted)
|
||||
expect(result.matchedCount).toBe(1);
|
||||
|
||||
// Verify updated values
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
|
||||
expect(doc.total_available_today).toBe(12);
|
||||
|
||||
// Verify only one document exists for this date
|
||||
const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. Summary includes new_units, rented_units, stale_units arrays
|
||||
// ---------------------------------------------------------------
|
||||
describe('summary arrays', () => {
|
||||
it('should store new_units, rented_units, and stale_units arrays', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201', 'B202', 'B203'],
|
||||
staleUnitsCount: 3,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual(['A101', 'A102']);
|
||||
expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
|
||||
expect(doc.stale_units).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle empty arrays for new_units and rented_units', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 25
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units).toEqual([]);
|
||||
expect(doc.rented_units).toEqual([]);
|
||||
expect(doc.stale_units).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Count fields are calculated correctly
|
||||
// ---------------------------------------------------------------
|
||||
describe('count fields', () => {
|
||||
it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 5,
|
||||
totalAvailable: 30
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units_count).toBe(3);
|
||||
expect(doc.rented_units_count).toBe(1);
|
||||
expect(doc.stale_units_count).toBe(5);
|
||||
});
|
||||
|
||||
it('should set all count fields to 0 when arrays are empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.new_units_count).toBe(0);
|
||||
expect(doc.rented_units_count).toBe(0);
|
||||
expect(doc.stale_units_count).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. net_change = newUnits.length - rentedUnits.length
|
||||
// ---------------------------------------------------------------
|
||||
describe('net_change calculation', () => {
|
||||
it('should calculate positive net_change when more units added than rented', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102', 'A103'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(2); // 3 - 1 = 2
|
||||
});
|
||||
|
||||
it('should calculate negative net_change when more units rented than added', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201', 'B202', 'B203', 'B204'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(-3); // 1 - 4 = -3
|
||||
});
|
||||
|
||||
it('should calculate zero net_change when equal new and rented', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201', 'B202'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.net_change).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. total_available_today is set correctly
|
||||
// ---------------------------------------------------------------
|
||||
describe('total_available_today', () => {
|
||||
it('should store the totalAvailable value as total_available_today', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 42
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_today).toBe(42);
|
||||
});
|
||||
|
||||
it('should handle totalAvailable of 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 0
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_today).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. total_available_yesterday fetched from previous day
|
||||
// ---------------------------------------------------------------
|
||||
describe('total_available_yesterday', () => {
|
||||
it('should fetch total_available_today from previous day summary', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 35,
|
||||
new_units: [],
|
||||
rented_units: []
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 35
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_yesterday).toBe(35);
|
||||
});
|
||||
|
||||
it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 20
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.total_available_yesterday).toBe(0);
|
||||
});
|
||||
|
||||
it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert March 31 summary
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-03-31',
|
||||
total_available_today: 50
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-04-01',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 48
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
|
||||
expect(doc.total_available_yesterday).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
|
||||
// ---------------------------------------------------------------
|
||||
describe('turnover_rate calculation', () => {
|
||||
it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 50 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 50
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 46
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
// 5 rented / 50 yesterday = 10%
|
||||
expect(doc.turnover_rate).toBe(10);
|
||||
});
|
||||
|
||||
it('should round turnover_rate to 2 decimal places', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 30 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 30
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: ['B201', 'B202'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 28
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
// 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
|
||||
expect(doc.turnover_rate).toBe(6.67);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. turnover_rate is 0 when yesterdayTotal is 0
|
||||
// ---------------------------------------------------------------
|
||||
describe('turnover_rate edge cases', () => {
|
||||
it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 15
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
|
||||
it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Insert yesterday's summary with 0 total available
|
||||
await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
|
||||
date: '2026-02-04',
|
||||
total_available_today: 0
|
||||
});
|
||||
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
|
||||
it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.turnover_rate).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 9. Error handling when updateOne fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw and log error when updateOne fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101'],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
// Create a mock db that throws on updateOne
|
||||
const mockCollection = {
|
||||
findOne: jest.fn().mockResolvedValue(null),
|
||||
updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
await expect(
|
||||
updateDailySummary(mockDb, summaryData, logger)
|
||||
).rejects.toThrow('Write concern timeout');
|
||||
|
||||
// Verify error was logged
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to update daily summary',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 10. Logging of summary update details
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log summary update details on success', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: ['A101', 'A102'],
|
||||
rentedUnits: ['B201'],
|
||||
staleUnitsCount: 1,
|
||||
totalAvailable: 18
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Daily summary updated',
|
||||
expect.objectContaining({
|
||||
date: '2026-02-05',
|
||||
newUnits: 2,
|
||||
rentedUnits: 1,
|
||||
totalAvailable: 18
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should include timestamp field in the stored summary', async () => {
|
||||
const logger = createMockLogger();
|
||||
const summaryData = {
|
||||
date: '2026-02-05',
|
||||
newUnits: [],
|
||||
rentedUnits: [],
|
||||
staleUnitsCount: 0,
|
||||
totalAvailable: 10
|
||||
};
|
||||
|
||||
await updateDailySummary(db, summaryData, logger);
|
||||
|
||||
const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
|
||||
expect(doc.timestamp).toBeDefined();
|
||||
expect(typeof doc.timestamp).toBe('string');
|
||||
// Verify it is a valid ISO timestamp
|
||||
expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user