SCRAPE-8: Implement markStaleUnits() (#13)
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com> Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
449
__tests__/scraper/markStaleUnits.test.js
Normal file
449
__tests__/scraper/markStaleUnits.test.js
Normal file
@ -0,0 +1,449 @@
|
||||
/**
|
||||
* Tests for markStaleUnits() availability tracking
|
||||
*
|
||||
* Covers:
|
||||
* - updateMany is called with correct filter (unit_code $nin currentCodes AND available: true)
|
||||
* - $set updates available to false and sets marked_stale_date
|
||||
* - Units in currentCodes are NOT marked stale
|
||||
* - Already unavailable units are not modified
|
||||
* - Error handling when updateMany fails
|
||||
* - Logging of modified count
|
||||
* - Empty currentUnitCodes handles gracefully
|
||||
*/
|
||||
|
||||
const { MongoClient } = require('mongodb');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
|
||||
// Will require markStaleUnits after implementation
|
||||
let markStaleUnits;
|
||||
|
||||
let mongoServer;
|
||||
let client;
|
||||
let db;
|
||||
|
||||
const UNITS_COLLECTION = 'units_migration_test';
|
||||
|
||||
// Mock logger for capturing log calls
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
const uri = mongoServer.getUri();
|
||||
client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
db = client.db('test_apartments');
|
||||
|
||||
// Dynamically require to pick up implementation
|
||||
({ markStaleUnits } = require('../../services/scraperService'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (client) await client.close();
|
||||
if (mongoServer) await mongoServer.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean collections before each test
|
||||
const collections = await db.listCollections().toArray();
|
||||
for (const col of collections) {
|
||||
await db.collection(col.name).deleteMany({});
|
||||
}
|
||||
});
|
||||
|
||||
describe('markStaleUnits', () => {
|
||||
// ---------------------------------------------------------------
|
||||
// 1. updateMany is called with correct filter
|
||||
// (unit_code $nin currentCodes AND available: true)
|
||||
// ---------------------------------------------------------------
|
||||
describe('filter criteria', () => {
|
||||
it('should call updateMany with $nin filter for currentUnitCodes and available: true', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
// Seed units in the database
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'A101', available: true, price: 1200 },
|
||||
{ unit_code: 'A102', available: true, price: 1300 },
|
||||
{ unit_code: 'B201', available: true, price: 1500 }
|
||||
]);
|
||||
|
||||
// Current scrape found A101 and A102, but NOT B201
|
||||
const currentCodes = new Set(['A101', 'A102']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// B201 should be marked stale (it was available but not in current scrape)
|
||||
expect(result.modifiedCount).toBe(1);
|
||||
|
||||
// Verify B201 was updated
|
||||
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'B201' });
|
||||
expect(staleUnit.available).toBe(false);
|
||||
expect(staleUnit.marked_stale_date).toBe(date);
|
||||
});
|
||||
|
||||
it('should only affect units NOT in the currentUnitCodes set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'X1', available: true },
|
||||
{ unit_code: 'X2', available: true },
|
||||
{ unit_code: 'X3', available: true },
|
||||
{ unit_code: 'X4', available: true }
|
||||
]);
|
||||
|
||||
// Current scrape found X1 and X3
|
||||
const currentCodes = new Set(['X1', 'X3']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// X1 and X3 should still be available
|
||||
const x1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X1' });
|
||||
const x3 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X3' });
|
||||
expect(x1.available).toBe(true);
|
||||
expect(x3.available).toBe(true);
|
||||
|
||||
// X2 and X4 should be marked stale
|
||||
const x2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X2' });
|
||||
const x4 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'X4' });
|
||||
expect(x2.available).toBe(false);
|
||||
expect(x4.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 2. $set updates available to false and sets marked_stale_date
|
||||
// ---------------------------------------------------------------
|
||||
describe('$set fields', () => {
|
||||
it('should set available to false for stale units', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'S100', available: true, price: 1000 },
|
||||
{ unit_code: 'S200', available: true, price: 1100 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['S100']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const staleUnit = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'S200' });
|
||||
expect(staleUnit.available).toBe(false);
|
||||
});
|
||||
|
||||
it('should set marked_stale_date to the provided date', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'D100', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'D100' });
|
||||
expect(doc.marked_stale_date).toBe('2026-02-05');
|
||||
});
|
||||
|
||||
it('should update both available and marked_stale_date in the same operation', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'C100', available: true, price: 900 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-01-15';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
const doc = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'C100' });
|
||||
expect(doc.available).toBe(false);
|
||||
expect(doc.marked_stale_date).toBe('2026-01-15');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 3. Units in currentCodes are NOT marked stale
|
||||
// ---------------------------------------------------------------
|
||||
describe('units in currentCodes not affected', () => {
|
||||
it('should not modify units that are in the currentUnitCodes set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'KEEP-1', available: true, price: 1200 },
|
||||
{ unit_code: 'KEEP-2', available: true, price: 1300 },
|
||||
{ unit_code: 'STALE-1', available: true, price: 1400 }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['KEEP-1', 'KEEP-2']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// Kept units should retain their original state
|
||||
const keep1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-1' });
|
||||
const keep2 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'KEEP-2' });
|
||||
|
||||
expect(keep1.available).toBe(true);
|
||||
expect(keep1.marked_stale_date).toBeUndefined();
|
||||
expect(keep2.available).toBe(true);
|
||||
expect(keep2.marked_stale_date).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return modifiedCount of 0 when all units are in currentCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'ALL-1', available: true },
|
||||
{ unit_code: 'ALL-2', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['ALL-1', 'ALL-2']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 4. Already unavailable units are not modified
|
||||
// ---------------------------------------------------------------
|
||||
describe('already unavailable units', () => {
|
||||
it('should not modify units that are already unavailable (available: false)', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'UNAVAIL-1', available: false, marked_stale_date: '2026-01-01' },
|
||||
{ unit_code: 'UNAVAIL-2', available: false },
|
||||
{ unit_code: 'AVAIL-1', available: true }
|
||||
]);
|
||||
|
||||
// None of these units are in the current scrape
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// Only AVAIL-1 should be modified (the already-unavailable ones should not be)
|
||||
expect(result.modifiedCount).toBe(1);
|
||||
|
||||
// Verify the already-unavailable unit's date was NOT changed
|
||||
const unavail1 = await db.collection(UNITS_COLLECTION).findOne({ unit_code: 'UNAVAIL-1' });
|
||||
expect(unavail1.marked_stale_date).toBe('2026-01-01');
|
||||
});
|
||||
|
||||
it('should return modifiedCount of 0 when no available units exist outside currentCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'U1', available: false },
|
||||
{ unit_code: 'U2', available: false }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set([]);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 5. Error handling when updateMany fails
|
||||
// ---------------------------------------------------------------
|
||||
describe('error handling', () => {
|
||||
it('should throw error when updateMany fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(new Error('Connection lost'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toThrow('Connection lost');
|
||||
});
|
||||
|
||||
it('should log error details when updateMany fails', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(new Error('Write concern timeout'))
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
try {
|
||||
await markStaleUnits(mockDb, currentCodes, date, logger);
|
||||
} catch (e) {
|
||||
// Expected to throw
|
||||
}
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to mark stale units',
|
||||
expect.objectContaining({
|
||||
errorType: 'Error',
|
||||
errorMessage: 'Write concern timeout'
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should re-throw the original error', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const originalError = new TypeError('Invalid operation');
|
||||
const mockCollection = {
|
||||
updateMany: jest.fn().mockRejectedValue(originalError)
|
||||
};
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection)
|
||||
};
|
||||
|
||||
const currentCodes = new Set(['A1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await expect(markStaleUnits(mockDb, currentCodes, date, logger)).rejects.toBe(originalError);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 6. Logging of modified count
|
||||
// ---------------------------------------------------------------
|
||||
describe('logging', () => {
|
||||
it('should log the count of stale units marked', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'LOG-1', available: true },
|
||||
{ unit_code: 'LOG-2', available: true },
|
||||
{ unit_code: 'LOG-3', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['LOG-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 2 })
|
||||
);
|
||||
});
|
||||
|
||||
it('should log count of 0 when no units are marked stale', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'ZERO-1', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['ZERO-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 0 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 7. Empty currentUnitCodes handles gracefully
|
||||
// ---------------------------------------------------------------
|
||||
describe('empty currentUnitCodes', () => {
|
||||
it('should mark all available units as stale when currentUnitCodes is empty', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'EMPTY-1', available: true },
|
||||
{ unit_code: 'EMPTY-2', available: true },
|
||||
{ unit_code: 'EMPTY-3', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(result.modifiedCount).toBe(3);
|
||||
|
||||
// All should be marked unavailable
|
||||
const docs = await db.collection(UNITS_COLLECTION).find({}).toArray();
|
||||
for (const doc of docs) {
|
||||
expect(doc.available).toBe(false);
|
||||
expect(doc.marked_stale_date).toBe('2026-02-05');
|
||||
}
|
||||
});
|
||||
|
||||
it('should not throw when currentUnitCodes is an empty Set', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
// Should not throw even with empty collection and empty set
|
||||
await expect(markStaleUnits(db, currentCodes, date, logger)).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('should handle empty database with empty currentUnitCodes', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
const currentCodes = new Set();
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
expect(result.modifiedCount).toBe(0);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Stale units marked',
|
||||
expect.objectContaining({ count: 0 })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 8. Return value
|
||||
// ---------------------------------------------------------------
|
||||
describe('return value', () => {
|
||||
it('should return the MongoDB updateMany result', async () => {
|
||||
const logger = createMockLogger();
|
||||
|
||||
await db.collection(UNITS_COLLECTION).insertMany([
|
||||
{ unit_code: 'RET-1', available: true },
|
||||
{ unit_code: 'RET-2', available: true }
|
||||
]);
|
||||
|
||||
const currentCodes = new Set(['RET-1']);
|
||||
const date = '2026-02-05';
|
||||
|
||||
const result = await markStaleUnits(db, currentCodes, date, logger);
|
||||
|
||||
// updateMany result should have these standard properties
|
||||
expect(result).toHaveProperty('matchedCount');
|
||||
expect(result).toHaveProperty('modifiedCount');
|
||||
expect(result).toHaveProperty('acknowledged');
|
||||
expect(result.acknowledged).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user