Implement updateDailySummary() aggregation function

Add daily summary aggregation that tracks apartment inventory changes:
- Upsert summaries by date key for idempotent operation
- Track new_units, rented_units, and stale_units arrays with counts
- Calculate net_change (new - rented) and total_available_today
- Fetch previous day summary for total_available_yesterday
- Compute turnover_rate as percentage of rented vs yesterday available
- Handle edge cases: zero division, missing previous day, month boundaries
- Full test coverage with 22 tests across all functional areas

__tests__/scraper/updateDailySummary.test.js

@@ -0,0 +1,561 @@
+/**
+ * Tests for updateDailySummary() daily summary aggregation
+ *
+ * Covers:
+ * - Upsert by date key (YYYY-MM-DD)
+ * - Summary includes new_units, rented_units, stale_units arrays
+ * - Count fields are calculated correctly
+ * - net_change = newUnits.length - rentedUnits.length
+ * - total_available_today is set correctly
+ * - total_available_yesterday fetched from previous day
+ * - turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+ * - turnover_rate is 0 when yesterdayTotal is 0
+ * - Error handling when updateOne fails
+ * - Logging of summary update details
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require updateDailySummary after implementation
+let updateDailySummary;
+
+let mongoServer;
+let client;
+let db;
+
+const DAILY_SUMMARIES_COLLECTION = 'daily_summaries';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ updateDailySummary } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('updateDailySummary', () => {
+  // ---------------------------------------------------------------
+  // 1. Upsert by date key
+  // ---------------------------------------------------------------
+  describe('upsert by date key', () => {
+    it('should insert a new summary when no document exists for the date', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 2,
+        totalAvailable: 15
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have upserted (created new)
+      expect(result.upsertedCount).toBe(1);
+
+      // Verify the document in the database
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc).not.toBeNull();
+      expect(doc.date).toBe('2026-02-05');
+    });
+
+    it('should update an existing summary for the same date (idempotent upsert)', async () => {
+      const logger = createMockLogger();
+
+      // Insert initial summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-05',
+        new_units: ['A101'],
+        rented_units: [],
+        total_available_today: 10
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 12
+      };
+
+      const result = await updateDailySummary(db, summaryData, logger);
+
+      // Should have matched and modified (not upserted)
+      expect(result.matchedCount).toBe(1);
+
+      // Verify updated values
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102', 'A103']);
+      expect(doc.total_available_today).toBe(12);
+
+      // Verify only one document exists for this date
+      const count = await db.collection(DAILY_SUMMARIES_COLLECTION).countDocuments({ date: '2026-02-05' });
+      expect(count).toBe(1);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. Summary includes new_units, rented_units, stale_units arrays
+  // ---------------------------------------------------------------
+  describe('summary arrays', () => {
+    it('should store new_units, rented_units, and stale_units arrays', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202', 'B203'],
+        staleUnitsCount: 3,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual(['A101', 'A102']);
+      expect(doc.rented_units).toEqual(['B201', 'B202', 'B203']);
+      expect(doc.stale_units).toEqual([]);
+    });
+
+    it('should handle empty arrays for new_units and rented_units', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 25
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units).toEqual([]);
+      expect(doc.rented_units).toEqual([]);
+      expect(doc.stale_units).toEqual([]);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Count fields are calculated correctly
+  // ---------------------------------------------------------------
+  describe('count fields', () => {
+    it('should calculate new_units_count, rented_units_count, and stale_units_count', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 5,
+        totalAvailable: 30
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(3);
+      expect(doc.rented_units_count).toBe(1);
+      expect(doc.stale_units_count).toBe(5);
+    });
+
+    it('should set all count fields to 0 when arrays are empty', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.new_units_count).toBe(0);
+      expect(doc.rented_units_count).toBe(0);
+      expect(doc.stale_units_count).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. net_change = newUnits.length - rentedUnits.length
+  // ---------------------------------------------------------------
+  describe('net_change calculation', () => {
+    it('should calculate positive net_change when more units added than rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102', 'A103'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(2); // 3 - 1 = 2
+    });
+
+    it('should calculate negative net_change when more units rented than added', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(-3); // 1 - 4 = -3
+    });
+
+    it('should calculate zero net_change when equal new and rented', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.net_change).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. total_available_today is set correctly
+  // ---------------------------------------------------------------
+  describe('total_available_today', () => {
+    it('should store the totalAvailable value as total_available_today', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 42
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(42);
+    });
+
+    it('should handle totalAvailable of 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 0
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_today).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. total_available_yesterday fetched from previous day
+  // ---------------------------------------------------------------
+  describe('total_available_yesterday', () => {
+    it('should fetch total_available_today from previous day summary', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 35,
+        new_units: [],
+        rented_units: []
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 35
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(35);
+    });
+
+    it('should set total_available_yesterday to 0 when no previous day summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 20
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.total_available_yesterday).toBe(0);
+    });
+
+    it('should correctly calculate yesterday for dates crossing month boundaries', async () => {
+      const logger = createMockLogger();
+
+      // Insert March 31 summary
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-03-31',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-04-01',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 48
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-04-01' });
+      expect(doc.total_available_yesterday).toBe(50);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. turnover_rate calculation (rentedUnits / yesterdayTotal * 100)
+  // ---------------------------------------------------------------
+  describe('turnover_rate calculation', () => {
+    it('should calculate turnover_rate as (rentedUnits / yesterdayTotal * 100)', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 50 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 50
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201', 'B202', 'B203', 'B204', 'B205'],
+        staleUnitsCount: 0,
+        totalAvailable: 46
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 5 rented / 50 yesterday = 10%
+      expect(doc.turnover_rate).toBe(10);
+    });
+
+    it('should round turnover_rate to 2 decimal places', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 30 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 30
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: ['B201', 'B202'],
+        staleUnitsCount: 0,
+        totalAvailable: 28
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      // 2 rented / 30 yesterday = 6.666...% => rounded to 6.67
+      expect(doc.turnover_rate).toBe(6.67);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. turnover_rate is 0 when yesterdayTotal is 0
+  // ---------------------------------------------------------------
+  describe('turnover_rate edge cases', () => {
+    it('should set turnover_rate to 0 when no yesterday summary exists', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 15
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when yesterday total_available_today is 0', async () => {
+      const logger = createMockLogger();
+
+      // Insert yesterday's summary with 0 total available
+      await db.collection(DAILY_SUMMARIES_COLLECTION).insertOne({
+        date: '2026-02-04',
+        total_available_today: 0
+      });
+
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+
+    it('should set turnover_rate to 0 when no units were rented and yesterdayTotal is 0', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.turnover_rate).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 9. Error handling when updateOne fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw and log error when updateOne fails', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101'],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      // Create a mock db that throws on updateOne
+      const mockCollection = {
+        findOne: jest.fn().mockResolvedValue(null),
+        updateOne: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      await expect(
+        updateDailySummary(mockDb, summaryData, logger)
+      ).rejects.toThrow('Write concern timeout');
+
+      // Verify error was logged
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to update daily summary',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 10. Logging of summary update details
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log summary update details on success', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: ['A101', 'A102'],
+        rentedUnits: ['B201'],
+        staleUnitsCount: 1,
+        totalAvailable: 18
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Daily summary updated',
+        expect.objectContaining({
+          date: '2026-02-05',
+          newUnits: 2,
+          rentedUnits: 1,
+          totalAvailable: 18
+        })
+      );
+    });
+
+    it('should include timestamp field in the stored summary', async () => {
+      const logger = createMockLogger();
+      const summaryData = {
+        date: '2026-02-05',
+        newUnits: [],
+        rentedUnits: [],
+        staleUnitsCount: 0,
+        totalAvailable: 10
+      };
+
+      await updateDailySummary(db, summaryData, logger);
+
+      const doc = await db.collection(DAILY_SUMMARIES_COLLECTION).findOne({ date: '2026-02-05' });
+      expect(doc.timestamp).toBeDefined();
+      expect(typeof doc.timestamp).toBe('string');
+      // Verify it is a valid ISO timestamp
+      expect(new Date(doc.timestamp).toISOString()).toBe(doc.timestamp);
+    });
+  });
+});