SCRAPE-8: Implement markStaleUnits() (#13)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

__tests__/scraper/markStaleUnits.test.js

@@ -0,0 +1,449 @@
+/**
+ * Tests for markStaleUnits() availability tracking
+ *
+ * Covers:
+ * - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
+ * - $set updates available to false and sets marked_stale_date
+ * - Units in currentCodes are NOT marked stale
+ * - Already unavailable units are not modified
+ * - Error handling when updateMany fails
+ * - Logging of modified count
+ * - Empty currentUnitCodes handles gracefully
+ */
+
+const { MongoClient } = require('mongodb');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Will require markStaleUnits after implementation
+let markStaleUnits;
+
+let mongoServer;
+let client;
+let db;
+
+const UNITS_COLLECTION = 'units_migration_test';
+
+// Mock logger for capturing log calls
+function createMockLogger() {
+  return {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+  };
+}
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  const uri = mongoServer.getUri();
+  client = new MongoClient(uri);
+  await client.connect();
+  db = client.db('test_apartments');
+
+  // Dynamically require to pick up implementation
+  ({ markStaleUnits } = require('../../services/scraperService'));
+});
+
+afterAll(async () => {
+  if (client) await client.close();
+  if (mongoServer) await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  // Clean collections before each test
+  const collections = await db.listCollections().toArray();
+  for (const col of collections) {
+    await db.collection(col.name).deleteMany({});
+  }
+});
+
+describe('markStaleUnits', () => {
+  // ---------------------------------------------------------------
+  // 1. updateMany is called with correct filter
+  //    (unit_code $nin currentCodes AND available: true)
+  // ---------------------------------------------------------------
+  describe('filter criteria', () => {
+    it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
+      const logger = createMockLogger();
+
+      // Seed units in the database
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'A101', available: true, price: 1200 },
+        { unit_code: 'A102', available: true, price: 1300 },
+        { unit_code: 'B201', available: true, price: 1500 }
+      ]);
+
+      // Current scrape found A101 and A102, but NOT B201
+      const currentCodes = new Set(['A101', 'A102']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // B201 should be marked stale (it was available but not in current scrape)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify B201 was updated
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
+      expect(staleUnit.available).toBe(false);
+      expect(staleUnit.marked_stale_date).toBe(date);
+    });
+
+    it('should only affect units NOT in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'X1', available: true },
+        { unit_code: 'X2', available: true },
+        { unit_code: 'X3', available: true },
+        { unit_code: 'X4', available: true }
+      ]);
+
+      // Current scrape found X1 and X3
+      const currentCodes = new Set(['X1', 'X3']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // X1 and X3 should still be available
+      const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
+      const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
+      expect(x1.available).toBe(true);
+      expect(x3.available).toBe(true);
+
+      // X2 and X4 should be marked stale
+      const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
+      const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
+      expect(x2.available).toBe(false);
+      expect(x4.available).toBe(false);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 2. $set updates available to false and sets marked_stale_date
+  // ---------------------------------------------------------------
+  describe('$set fields', () => {
+    it('should set available to false for stale units', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'S100', available: true, price: 1000 },
+        { unit_code: 'S200', available: true, price: 1100 }
+      ]);
+
+      const currentCodes = new Set(['S100']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
+      expect(staleUnit.available).toBe(false);
+    });
+
+    it('should set marked_stale_date to the provided date', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'D100', available: true }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
+      expect(doc.marked_stale_date).toBe('2026-02-05');
+    });
+
+    it('should update both available and marked_stale_date in the same operation', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'C100', available: true, price: 900 }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-01-15';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
+      expect(doc.available).toBe(false);
+      expect(doc.marked_stale_date).toBe('2026-01-15');
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 3. Units in currentCodes are NOT marked stale
+  // ---------------------------------------------------------------
+  describe('units in currentCodes not affected', () => {
+    it('should not modify units that are in the currentUnitCodes set', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'KEEP-1', available: true, price: 1200 },
+        { unit_code: 'KEEP-2', available: true, price: 1300 },
+        { unit_code: 'STALE-1', available: true, price: 1400 }
+      ]);
+
+      const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      // Kept units should retain their original state
+      const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
+      const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
+
+      expect(keep1.available).toBe(true);
+      expect(keep1.marked_stale_date).toBeUndefined();
+      expect(keep2.available).toBe(true);
+      expect(keep2.marked_stale_date).toBeUndefined();
+    });
+
+    it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ALL-1', available: true },
+        { unit_code: 'ALL-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['ALL-1', 'ALL-2']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 4. Already unavailable units are not modified
+  // ---------------------------------------------------------------
+  describe('already unavailable units', () => {
+    it('should not modify units that are already unavailable (available: false)', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
+        { unit_code: 'UNAVAIL-2', available: false },
+        { unit_code: 'AVAIL-1', available: true }
+      ]);
+
+      // None of these units are in the current scrape
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // Only AVAIL-1 should be modified (the already-unavailable ones should not be)
+      expect(result.modifiedCount).toBe(1);
+
+      // Verify the already-unavailable unit's date was NOT changed
+      const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
+      expect(unavail1.marked_stale_date).toBe('2026-01-01');
+    });
+
+    it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'U1', available: false },
+        { unit_code: 'U2', available: false }
+      ]);
+
+      const currentCodes = new Set([]);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+      expect(result.modifiedCount).toBe(0);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 5. Error handling when updateMany fails
+  // ---------------------------------------------------------------
+  describe('error handling', () => {
+    it('should throw error when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
+    });
+
+    it('should log error details when updateMany fails', async () => {
+      const logger = createMockLogger();
+
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      try {
+        await markStaleUnits(mockDb, currentCodes, date, logger);
+      } catch (e) {
+        // Expected to throw
+      }
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to mark stale units',
+        expect.objectContaining({
+          errorType: 'Error',
+          errorMessage: 'Write concern timeout'
+        })
+      );
+    });
+
+    it('should re-throw the original error', async () => {
+      const logger = createMockLogger();
+
+      const originalError = new TypeError('Invalid operation');
+      const mockCollection = {
+        updateMany: jest.fn().mockRejectedValue(originalError)
+      };
+      const mockDb = {
+        collection: jest.fn().mockReturnValue(mockCollection)
+      };
+
+      const currentCodes = new Set(['A1']);
+      const date = '2026-02-05';
+
+      await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 6. Logging of modified count
+  // ---------------------------------------------------------------
+  describe('logging', () => {
+    it('should log the count of stale units marked', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'LOG-1', available: true },
+        { unit_code: 'LOG-2', available: true },
+        { unit_code: 'LOG-3', available: true }
+      ]);
+
+      const currentCodes = new Set(['LOG-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 2 })
+      );
+    });
+
+    it('should log count of 0 when no units are marked stale', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'ZERO-1', available: true }
+      ]);
+
+      const currentCodes = new Set(['ZERO-1']);
+      const date = '2026-02-05';
+
+      await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 7. Empty currentUnitCodes handles gracefully
+  // ---------------------------------------------------------------
+  describe('empty currentUnitCodes', () => {
+    it('should mark all available units as stale when currentUnitCodes is empty', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'EMPTY-1', available: true },
+        { unit_code: 'EMPTY-2', available: true },
+        { unit_code: 'EMPTY-3', available: true }
+      ]);
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(3);
+
+      // All should be marked unavailable
+      const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
+      for (const doc of docs) {
+        expect(doc.available).toBe(false);
+        expect(doc.marked_stale_date).toBe('2026-02-05');
+      }
+    });
+
+    it('should not throw when currentUnitCodes is an empty Set', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      // Should not throw even with empty collection and empty set
+      await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
+    });
+
+    it('should handle empty database with empty currentUnitCodes', async () => {
+      const logger = createMockLogger();
+
+      const currentCodes = new Set();
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      expect(result.modifiedCount).toBe(0);
+      expect(logger.info).toHaveBeenCalledWith(
+        'Stale units marked',
+        expect.objectContaining({ count: 0 })
+      );
+    });
+  });
+
+  // ---------------------------------------------------------------
+  // 8. Return value
+  // ---------------------------------------------------------------
+  describe('return value', () => {
+    it('should return the MongoDB updateMany result', async () => {
+      const logger = createMockLogger();
+
+      await db.collection(UNITS_COLLECTION).insertMany([
+        { unit_code: 'RET-1', available: true },
+        { unit_code: 'RET-2', available: true }
+      ]);
+
+      const currentCodes = new Set(['RET-1']);
+      const date = '2026-02-05';
+
+      const result = await markStaleUnits(db, currentCodes, date, logger);
+
+      // updateMany result should have these standard properties
+      expect(result).toHaveProperty('matchedCount');
+      expect(result).toHaveProperty('modifiedCount');
+      expect(result).toHaveProperty('acknowledged');
+      expect(result.acknowledged).toBe(true);
+    });
+  });
+});

services/scraperService.js

@@ -493,12 +493,55 @@ async function insertPrices(db, units, date, logger) {
   }
 }
 
+/**
+ * Mark units not in current scrape as stale/unavailable.
+ * Uses updateMany to set available: false and marked_stale_date
+ * for all units whose unit_code is NOT in the current scrape
+ * and that are currently available.
+ *
+ * @param {Db} db - MongoDB database instance
+ * @param {Set<string>} currentUnitCodes - Unit codes from current scrape
+ * @param {string} date - Current date in YYYY-MM-DD format
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<Object>} Update result
+ */
+async function markStaleUnits(db, currentUnitCodes, date, logger) {
+  const collection = db.collection(config.COLLECTIONS.UNITS);
+
+  try {
+    const result = await collection.updateMany(
+      {
+        unit_code: { $nin: [...currentUnitCodes] },
+        available: true
+      },
+      {
+        $set: {
+          available: false,
+          marked_stale_date: date
+        }
+      }
+    );
+
+    logger.info('Stale units marked', { count: result.modifiedCount });
+
+    return result;
+
+  } catch (error) {
+    logger.error('Failed to mark stale units', {
+      errorType: error.name,
+      errorMessage: error.message
+    });
+    throw error;
+  }
+}
+
 module.exports = {
   fetchPage,
   parseUnits,
   convertDataTypes,
   upsertUnits,
   insertPrices,
+  markStaleUnits,
   // Export helpers for testing
   parseInteger,
   parsePositiveInteger,