Add HTML fixture files for scraper unit tests

Create three HTML fixture files in __tests__/scraper/fixtures/ to support
deterministic testing of the scraper's HTML parsing logic:

- sample-listing.html: Contains 10 real unit articles extracted from the
  live listings page, covering studios, 1BR, and 2BR floor plans with
  varied pricing and availability dates.

- sample-listing-empty.html: Minimal page structure with an empty units
  section, for testing graceful handling of pages with no listings.

- sample-listing-call.html: Contains units with "Call for pricing" instead
  of numeric rent values, for testing the parser's handling of non-numeric
  price fields.

.github/workflows/deploy.yml

@@ -44,10 +44,8 @@ jobs:
           set +e
           OUTPUT=$(npm run lint 2>&1)
           EXIT_CODE=$?
-          # Truncate before writing to GITHUB_OUTPUT to prevent
-          # "argument list too long" in downstream jobs
           echo "lint_output<<EOF" >> $GITHUB_OUTPUT
-          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
           exit $EXIT_CODE
 
@@ -58,10 +56,8 @@ jobs:
           set +e
           OUTPUT=$(npm test -- --runInBand 2>&1)
           EXIT_CODE=$?
-          # Truncate before writing to GITHUB_OUTPUT to prevent
-          # "argument list too long" in downstream jobs
           echo "test_output<<EOF" >> $GITHUB_OUTPUT
-          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
           exit $EXIT_CODE
         env:
@@ -102,9 +98,9 @@ jobs:
             OVERALL_STATUS="failure"
           fi
 
-          # Outputs are already truncated at the source (ci job)
+          # Truncate outputs if too long (max 10000 chars each)
-          LINT_OUTPUT="$RAW_LINT_OUTPUT"
+          LINT_OUTPUT="${RAW_LINT_OUTPUT:0:10000}"
-          TEST_OUTPUT="$RAW_TEST_OUTPUT"
+          TEST_OUTPUT="${RAW_TEST_OUTPUT:0:10000}"
 
           # Build JSON payload
           PAYLOAD=$(jq -n \
@@ -264,13 +260,12 @@ jobs:
       # ============================================================
       # SBOM Generation with Syft
       # ============================================================
-      - name: Install Syft
-        run: |
-          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
-
       - name: Generate SBOM with Syft
-        run: |
+        uses: anchore/sbom-action@v0
-          syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json
+        with:
+          image: ${{ steps.set-tag.outputs.full_image }}
+          format: spdx-json
+          output-file: sbom.spdx.json
 
       # ============================================================
       # Image Signing with Cosign

__tests__/scraper/insertPrices.test.js

@@ -22,7 +22,7 @@ let mongoServer;
 let client;
 let db;
 
-const PRICES_COLLECTION = 'unit_prices_scraper';
+const PRICES_COLLECTION = 'unit_prices_migration_test';
 
 // Mock logger for capturing log calls
 function createMockLogger() {

__tests__/scraper/markStaleUnits.test.js

@@ -21,7 +21,7 @@ let mongoServer;
 let client;
 let db;
 
-const UNITS_COLLECTION = 'units_scraper';
+const UNITS_COLLECTION = 'units_migration_test';
 
 // Mock logger for capturing log calls
 function createMockLogger() {

__tests__/scraper/parseUnits.test.js

@@ -87,55 +87,55 @@ describe('parseUnits', () => {
   describe('attribute extraction', () => {
     it('should extract all expected attributes from a fully populated article', () => {
       const units = parseUnits(sampleHtml, mockLogger);
-      const unit1 = units.find(u => u.unit_code === 'W2707');
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
 
       expect(unit1).toBeDefined();
 
       // Core identifiers
-      expect(unit1.id).toBe('154842');
+      expect(unit1.id).toBe('1001');
-      expect(unit1.unit_code).toBe('W2707');
+      expect(unit1.unit_code).toBe('CCT-101');
-      expect(unit1.unit_id).toBe('154842');
+      expect(unit1.unit_id).toBe('5001');
 
       // Physical attributes
-      expect(unit1.floor).toBe('2755');
+      expect(unit1.floor).toBe('1');
-      expect(unit1.area).toBe('1210');
+      expect(unit1.area).toBe('750');
-      expect(unit1.bed_count).toBe('2');
+      expect(unit1.bed_count).toBe('1');
-      expect(unit1.bath_count).toBe('2');
+      expect(unit1.bath_count).toBe('1');
 
       // Pricing
-      expect(unit1.price).toBe('5230');
+      expect(unit1.price).toBe('1450');
 
       // Availability
       expect(unit1.available).toBe('true');
       expect(unit1.unavailable).toBe('false');
-      expect(unit1.soonest).toBe('2025-10-11');
+      expect(unit1.soonest).toBe('2026-03-01');
-      expect(unit1.date_available).toBe('1760140800');
+      expect(unit1.date_available).toBe('20260301');
 
       // Plan information
-      expect(unit1.plan_id).toBe('10270');
+      expect(unit1.plan_id).toBe('201');
-      expect(unit1.plan_name).toBe('Pyramid Peak - Terrace');
+      expect(unit1.plan_name).toBe('Alpine');
 
       // Property information
       expect(unit1.obj_type).toBe('unit');
       expect(unit1.community).toBe('Country Club Towers');
-      expect(unit1.asset).toBe('420');
+      expect(unit1.asset).toBe('100');
 
       // URLs
-      expect(unit1.href).toBe('?spaces_tab=unit-detail&detail=154842');
+      expect(unit1.href).toBe('/units/CCT-101');
-      expect(unit1.inventory_href).toBe('?spaces_tab=unit-detail&detail=154842');
+      expect(unit1.inventory_href).toBe('/inventory/CCT-101');
 
       // Specials
-      expect(unit1.specials_content).toBe('');
+      expect(unit1.specials_content).toBe('First month free!');
     });
 
     it('should extract correct count of units from the fixture', () => {
       const units = parseUnits(sampleHtml, mockLogger);
-      expect(units).toHaveLength(10);
+      expect(units).toHaveLength(5);
     });
 
     it('should return raw string values without type conversion', () => {
       const units = parseUnits(sampleHtml, mockLogger);
-      const unit = units.find(u => u.unit_code === 'W2707');
+      const unit = units.find(u => u.unit_code === 'CCT-101');
 
       // All values should be strings (raw extraction, no conversion)
       expect(typeof unit.id).toBe('string');
@@ -152,48 +152,24 @@ describe('parseUnits', () => {
   // ---------------------------------------------------------------
   describe('image URL extraction', () => {
     it('should extract image_url from img src attribute', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      const unit1 = units.find(u => u.unit_code === 'CCT-101');
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-1" data-spaces-sort-price="1000">
-            <img src="https://example.com/images/unit-1.jpg" />
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
 
-      expect(units[0].image_url).toBe('https://example.com/images/unit-1.jpg');
+      expect(unit1.image_url).toBe('https://example.com/images/unit-101.jpg');
     });
 
     it('should fall back to data-src when src is not present', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-2" data-spaces-sort-price="1000">
-            <img data-src="https://example.com/images/unit-2.jpg" />
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
 
-      expect(units[0].image_url).toBe('https://example.com/images/unit-2.jpg');
+      expect(unit2.image_url).toBe('https://example.com/images/unit-205.jpg');
     });
 
     it('should not set image_url when no img element exists', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
-        <section class="spaces__tab-unit">
-          <article data-spaces-id="1" data-spaces-unit="IMG-3" data-spaces-sort-price="1000">
-            <div>No image here</div>
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
 
-      expect(units[0].image_url).toBeUndefined();
+      expect(unit3.image_url).toBeUndefined();
     });
   });
 
@@ -202,67 +178,37 @@ describe('parseUnits', () => {
   // ---------------------------------------------------------------
   describe('missing attributes', () => {
     it('should return undefined for attributes not present on the article', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      // Unit 5 (CCT-520) is missing floor, area, bed_count, bath_count,
-        <section class="spaces__tab-unit">
+      // soonest, date_available, plan_id, plan_name, href, inventory_href, specials_content
-          <article
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
-            data-spaces-id="1005"
-            data-spaces-unit="SPARSE-1"
-            data-spaces-unit-id="5005"
-            data-spaces-sort-price="1300"
-            data-spaces-available="true"
-            data-spaces-obj="unit"
-            data-spaces-community="Country Club Towers"
-            data-spaces-asset="100">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
 
-      expect(unit).toBeDefined();
+      expect(unit5).toBeDefined();
-      expect(unit.floor).toBeUndefined();
+      expect(unit5.floor).toBeUndefined();
-      expect(unit.area).toBeUndefined();
+      expect(unit5.area).toBeUndefined();
-      expect(unit.bed_count).toBeUndefined();
+      expect(unit5.bed_count).toBeUndefined();
-      expect(unit.bath_count).toBeUndefined();
+      expect(unit5.bath_count).toBeUndefined();
-      expect(unit.soonest).toBeUndefined();
+      expect(unit5.soonest).toBeUndefined();
-      expect(unit.date_available).toBeUndefined();
+      expect(unit5.date_available).toBeUndefined();
-      expect(unit.plan_id).toBeUndefined();
+      expect(unit5.plan_id).toBeUndefined();
-      expect(unit.plan_name).toBeUndefined();
+      expect(unit5.plan_name).toBeUndefined();
-      expect(unit.href).toBeUndefined();
+      expect(unit5.href).toBeUndefined();
-      expect(unit.inventory_href).toBeUndefined();
+      expect(unit5.inventory_href).toBeUndefined();
-      expect(unit.specials_content).toBeUndefined();
+      expect(unit5.specials_content).toBeUndefined();
     });
 
     it('should still extract the attributes that are present on a sparse article', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      const unit5 = units.find(u => u.unit_code === 'CCT-520');
-        <section class="spaces__tab-unit">
-          <article
-            data-spaces-id="1005"
-            data-spaces-unit="SPARSE-1"
-            data-spaces-unit-id="5005"
-            data-spaces-sort-price="1300"
-            data-spaces-available="true"
-            data-spaces-obj="unit"
-            data-spaces-community="Country Club Towers"
-            data-spaces-asset="100">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
 
-      expect(unit.id).toBe('1005');
+      expect(unit5.id).toBe('1005');
-      expect(unit.unit_code).toBe('SPARSE-1');
+      expect(unit5.unit_code).toBe('CCT-520');
-      expect(unit.unit_id).toBe('5005');
+      expect(unit5.unit_id).toBe('5005');
-      expect(unit.price).toBe('1300');
+      expect(unit5.price).toBe('1300');
-      expect(unit.available).toBe('true');
+      expect(unit5.available).toBe('true');
-      expect(unit.obj_type).toBe('unit');
+      expect(unit5.obj_type).toBe('unit');
-      expect(unit.community).toBe('Country Club Towers');
+      expect(unit5.community).toBe('Country Club Towers');
-      expect(unit.asset).toBe('100');
+      expect(unit5.asset).toBe('100');
     });
   });
 
@@ -367,7 +313,7 @@ describe('parseUnits', () => {
 
       expect(mockLogger.info).toHaveBeenCalledWith(
         'Units parsed',
-        expect.objectContaining({ count: 10 })
+        expect.objectContaining({ count: 5 })
       );
     });
 
@@ -384,134 +330,31 @@ describe('parseUnits', () => {
   });
 
   // ---------------------------------------------------------------
-  // 7. Fixture-based tests (sample-listing-empty.html, sample-listing-call.html)
+  // 7. Multiple units with varied data
-  // ---------------------------------------------------------------
-  describe('fixture: sample-listing-empty.html', () => {
-    let emptyHtml;
-
-    beforeAll(() => {
-      emptyHtml = fs.readFileSync(
-        path.join(fixturesDir, 'sample-listing-empty.html'),
-        'utf-8'
-      );
-    });
-
-    it('should return empty array when fixture has container but no articles', () => {
-      const units = parseUnits(emptyHtml, mockLogger);
-
-      expect(units).toEqual([]);
-      // Container exists so no error about missing container
-      expect(mockLogger.error).not.toHaveBeenCalled();
-    });
-
-    it('should log zero parsed units for empty fixture', () => {
-      parseUnits(emptyHtml, mockLogger);
-
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Units parsed',
-        expect.objectContaining({ count: 0 })
-      );
-    });
-  });
-
-  describe('fixture: sample-listing-call.html', () => {
-    let callHtml;
-
-    beforeAll(() => {
-      callHtml = fs.readFileSync(
-        path.join(fixturesDir, 'sample-listing-call.html'),
-        'utf-8'
-      );
-    });
-
-    it('should extract both units from the call-for-pricing fixture', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units).toHaveLength(2);
-      expect(units.map(u => u.unit_code)).toEqual(['P-101', 'P-102']);
-    });
-
-    it('should preserve "Call for pricing" as raw string in the price field', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units[0].price).toBe('Call for pricing');
-      expect(units[1].price).toBe('Call for pricing');
-    });
-
-    it('should extract all attributes correctly from call-for-pricing units', () => {
-      const units = parseUnits(callHtml, mockLogger);
-      const unit = units.find(u => u.unit_code === 'P-101');
-
-      expect(unit.id).toBe('9001');
-      expect(unit.unit_id).toBe('9001');
-      expect(unit.floor).toBe('1');
-      expect(unit.area).toBe('750');
-      expect(unit.bed_count).toBe('1');
-      expect(unit.bath_count).toBe('1');
-      expect(unit.available).toBe('true');
-      expect(unit.unavailable).toBe('false');
-      expect(unit.soonest).toBe('Now');
-      expect(unit.date_available).toBe('1706745600');
-      expect(unit.plan_id).toBe('500');
-      expect(unit.plan_name).toBe('Penthouse A');
-      expect(unit.community).toBe('Country Club Towers');
-      expect(unit.asset).toBe('420');
-      expect(unit.href).toBe('?spaces_tab=unit-detail&detail=9001');
-      expect(unit.inventory_href).toBe('?spaces_tab=unit-detail&detail=9001');
-      expect(unit.specials_content).toBe('');
-    });
-
-    it('should not have image_url when fixture units lack img elements', () => {
-      const units = parseUnits(callHtml, mockLogger);
-
-      expect(units[0].image_url).toBeUndefined();
-      expect(units[1].image_url).toBeUndefined();
-    });
-  });
-
-  // ---------------------------------------------------------------
-  // 8. Multiple units with varied data
   // ---------------------------------------------------------------
   describe('varied unit data', () => {
     it('should handle unavailable units correctly', () => {
-      const html = `
+      const units = parseUnits(sampleHtml, mockLogger);
-        <html><body>
+      const unit4 = units.find(u => u.unit_code === 'CCT-415');
-        <section class="spaces__tab-unit">
-          <article
-            data-spaces-id="4001"
-            data-spaces-unit="UNAVAIL-1"
-            data-spaces-sort-price="2000"
-            data-spaces-available="false"
-            data-spaces-unavailable="true"
-            data-spaces-soonest=""
-            data-spaces-sort-date="">
-          </article>
-        </section>
-        </body></html>
-      `;
-      const units = parseUnits(html, mockLogger);
-      const unit = units[0];
 
-      expect(unit.available).toBe('false');
+      expect(unit4.available).toBe('false');
-      expect(unit.unavailable).toBe('true');
+      expect(unit4.unavailable).toBe('true');
-      expect(unit.soonest).toBe('');
+      expect(unit4.soonest).toBe('');
-      expect(unit.date_available).toBe('');
+      expect(unit4.date_available).toBe('');
     });
 
     it('should handle half bath counts', () => {
       const units = parseUnits(sampleHtml, mockLogger);
-      // E3205 is the penthouse with 2.5 baths
+      const unit2 = units.find(u => u.unit_code === 'CCT-205');
-      const unit = units.find(u => u.unit_code === 'E3205');
 
-      expect(unit.bath_count).toBe('2.5');
+      expect(unit2.bath_count).toBe('1.5');
     });
 
     it('should handle studio units (bed_count = 0)', () => {
       const units = parseUnits(sampleHtml, mockLogger);
-      // W2603 is the studio with bed_count 0
+      const unit3 = units.find(u => u.unit_code === 'CCT-310');
-      const unit = units.find(u => u.unit_code === 'W2603');
 
-      expect(unit.bed_count).toBe('0');
+      expect(unit3.bed_count).toBe('0');
     });
   });
 });

__tests__/scraper/scraperConfig.test.js

@@ -50,15 +50,15 @@ describe('config/scraper', () => {
   });
 
   describe('SCRAPER_TIMEZONE', () => {
-    test('should default to "America/Denver" when env var not set', () => {
+    test('should default to "UTC" when env var not set', () => {
       const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
     });
 
     test('should use env var override when set', () => {
-      process.env.SCRAPER_TIMEZONE = 'America/New_York';
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
       const config = require('../../config/scraper');
-      expect(config.SCRAPER_TIMEZONE).toBe('America/New_York');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
     });
   });
 
@@ -142,14 +142,14 @@ describe('config/scraper', () => {
 
   describe('COLLECTIONS', () => {
     describe('default values', () => {
-      test('UNITS should default to "units_scraper"', () => {
+      test('UNITS should default to "units_migration_test"', () => {
         const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.UNITS).toBe('units_scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
       });
 
-      test('PRICES should default to "unit_prices_scraper"', () => {
+      test('PRICES should default to "unit_prices_migration_test"', () => {
         const config = require('../../config/scraper');
-        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
       });
 
       test('DAILY_SUMMARIES should default to "daily_summaries"', () => {

__tests__/scraper/scraperRoutes.test.js

@@ -118,8 +118,8 @@ jest.mock('../../config/scraper', () => ({
   RETRY_CONFIG: { maxRetries: 3, baseDelay: 1000, timeout: 30000 },
   SHUTDOWN_TIMEOUT: 30000,
   COLLECTIONS: {
-    UNITS: 'units_scraper',
+    UNITS: 'units_migration_test',
-    PRICES: 'unit_prices_scraper',
+    PRICES: 'unit_prices_migration_test',
     DAILY_SUMMARIES: 'daily_summaries',
     SCRAPER_RUNS: 'scraper_runs'
   }

__tests__/scraper/scraperService.test.js

@@ -20,10 +20,9 @@ let mongoServer;
 let client;
 let db;
 
-// We will require runScrape, recordScraperRun, and createScraperIndexes after implementation
+// We will require runScrape and recordScraperRun after implementation
 let runScrape;
 let recordScraperRun;
-let createScraperIndexes;
 
 // Store original module references so we can mock individual functions
 let scraperService;
@@ -37,7 +36,7 @@ beforeAll(async () => {
 
   // Dynamically require to pick up implementation
   scraperService = require('../../services/scraperService');
-  ({ runScrape, recordScraperRun, createScraperIndexes } = scraperService);
+  ({ runScrape, recordScraperRun } = scraperService);
 });
 
 afterAll(async () => {
@@ -220,11 +219,11 @@ describe('runScrape', () => {
     expect(result.staleUnitsCount).toBe(0);
 
     // Verify no units were written to the units collection
-    const unitsCount = await db.collection('units_scraper').countDocuments();
+    const unitsCount = await db.collection('units_migration_test').countDocuments();
     expect(unitsCount).toBe(0);
 
     // Verify no prices were written
-    const pricesCount = await db.collection('unit_prices_scraper').countDocuments();
+    const pricesCount = await db.collection('unit_prices_migration_test').countDocuments();
     expect(pricesCount).toBe(0);
   });
 
@@ -278,7 +277,7 @@ describe('runScrape', () => {
     const faultyDb = {
       collection: (name) => {
         const realCollection = db.collection(name);
-        if (name === 'units_scraper') {
+        if (name === 'units_migration_test') {
           return new Proxy(realCollection, {
             get(target, prop) {
               if (prop === 'bulkWrite') {
@@ -344,7 +343,7 @@ describe('runScrape', () => {
     const faultyDb = {
       collection: (name) => {
         const realCollection = db.collection(name);
-        if (name === 'unit_prices_scraper') {
+        if (name === 'unit_prices_migration_test') {
           return new Proxy(realCollection, {
             get(target, prop) {
               if (prop === 'bulkWrite') {
@@ -389,7 +388,7 @@ describe('runScrape', () => {
     const yesterdayStr = yesterday.toISOString().split('T')[0];
 
     // Yesterday had units: OLD-A, OLD-B, OLD-C
-    await db.collection('unit_prices_scraper').insertMany([
+    await db.collection('unit_prices_migration_test').insertMany([
       { unit_code: 'OLD-A', date_checked: yesterdayStr, price: 1000 },
       { unit_code: 'OLD-B', date_checked: yesterdayStr, price: 1100 },
       { unit_code: 'OLD-C', date_checked: yesterdayStr, price: 1200 }
@@ -632,7 +631,7 @@ describe('sanitizeError', () => {
       const faultyDb = {
         collection: (name) => {
           const realCollection = db.collection(name);
-          if (name === 'units_scraper') {
+          if (name === 'units_migration_test') {
             return new Proxy(realCollection, {
               get(target, prop) {
                 if (prop === 'bulkWrite') {
@@ -675,65 +674,3 @@ describe('sanitizeError', () => {
     });
   });
 });
-
-// ============================================================
-// Test: createScraperIndexes
-// ============================================================
-describe('createScraperIndexes', () => {
-  const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
-
-  beforeEach(() => {
-    mockLogger.info.mockClear();
-    mockLogger.warn.mockClear();
-    mockLogger.error.mockClear();
-  });
-
-  it('should create the status_startedAt compound index', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    const indexes = await db.collection('scraper_runs').indexes();
-    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
-    expect(statusIndex).toBeTruthy();
-    expect(statusIndex.key).toEqual({ status: 1, startedAt: -1 });
-  });
-
-  it('should create the startedAt_desc index', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    const indexes = await db.collection('scraper_runs').indexes();
-    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
-    expect(startedAtIndex).toBeTruthy();
-    expect(startedAtIndex.key).toEqual({ startedAt: -1 });
-  });
-
-  it('should log success after creating indexes', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    expect(mockLogger.info).toHaveBeenCalledWith('Scraper indexes created successfully');
-  });
-
-  it('should be idempotent (no error on second call)', async () => {
-    await createScraperIndexes(db, mockLogger);
-
-    // Calling again should not throw
-    await expect(createScraperIndexes(db, mockLogger)).resolves.not.toThrow();
-
-    // Indexes should still exist
-    const indexes = await db.collection('scraper_runs').indexes();
-    const statusIndex = indexes.find(idx => idx.name === 'status_startedAt');
-    const startedAtIndex = indexes.find(idx => idx.name === 'startedAt_desc');
-    expect(statusIndex).toBeTruthy();
-    expect(startedAtIndex).toBeTruthy();
-  });
-
-  it('should handle database errors gracefully', async () => {
-    // Pass an object that will throw when collection() is called
-    const faultyDb = {
-      collection: () => {
-        throw new Error('Connection lost');
-      }
-    };
-
-    await expect(createScraperIndexes(faultyDb, mockLogger)).rejects.toThrow('Connection lost');
-  });
-});

__tests__/scraper/serverIntegration.test.js

@@ -1,177 +0,0 @@
-/**
- * Tests for scraper integration in server.js startup
- *
- * Covers:
- * - Server initializes scraper scheduler on startup
- * - Server does not crash if scheduler init fails
- * - Indexes are created before scheduler starts
- * - Scheduler not initialized when SCRAPER_ENABLED=false
- *
- * Strategy: Instead of importing server.js directly (which starts Express and
- * connects to MongoDB), we replicate the scraper initialization logic from
- * connectToMongoDB() and test it with mocked dependencies. This verifies the
- * integration contract without needing a full server boot.
- */
-
-const { MongoClient } = require('mongodb');
-
-describe('Server Scraper Integration', () => {
-  let connection;
-  let db;
-
-  beforeAll(async () => {
-    const uri = process.env.MONGO_URI;
-    connection = await MongoClient.connect(uri);
-    db = connection.db('apartments_server_integration_test');
-  });
-
-  afterAll(async () => {
-    if (connection) {
-      await connection.close();
-    }
-  });
-
-  describe('scraper initialization during startup', () => {
-    let mockCreateScraperIndexes;
-    let mockInitializeScheduler;
-    let mockCreateLogger;
-    let mockLogger;
-
-    beforeEach(() => {
-      mockLogger = {
-        info: jest.fn(),
-        warn: jest.fn(),
-        error: jest.fn(),
-        debug: jest.fn()
-      };
-      mockCreateLogger = jest.fn(() => mockLogger);
-      mockCreateScraperIndexes = jest.fn().mockResolvedValue(undefined);
-      mockInitializeScheduler = jest.fn();
-    });
-
-    /**
-     * Replicates the scraper initialization logic from server.js connectToMongoDB().
-     * This is the exact pattern used in production:
-     *
-     *   try {
-     *     const scraperLogger = createLogger('server-init');
-     *     await createScraperIndexes(db, scraperLogger);
-     *     if (scraperConfig.SCRAPER_ENABLED) {
-     *       initializeScheduler(db);
-     *     }
-     *   } catch (error) {
-     *     console.error('Scraper initialization failed (server continues):', error.message);
-     *   }
-     */
-    async function runScraperInit(testDb, config) {
-      try {
-        const scraperLogger = mockCreateLogger('server-init');
-        await mockCreateScraperIndexes(testDb, scraperLogger);
-        if (config.SCRAPER_ENABLED) {
-          mockInitializeScheduler(testDb);
-        }
-        return { success: true };
-      } catch (error) {
-        return { success: false, error: error.message };
-      }
-    }
-
-    it('should initialize scraper scheduler on startup when enabled', async () => {
-      const config = { SCRAPER_ENABLED: true };
-
-      const result = await runScraperInit(db, config);
-
-      expect(result.success).toBe(true);
-      expect(mockInitializeScheduler).toHaveBeenCalledTimes(1);
-      expect(mockInitializeScheduler).toHaveBeenCalledWith(db);
-    });
-
-    it('should not crash if createScraperIndexes throws an error', async () => {
-      mockCreateScraperIndexes.mockRejectedValue(new Error('Index creation failed'));
-      const config = { SCRAPER_ENABLED: true };
-
-      const result = await runScraperInit(db, config);
-
-      // The server should continue (error is caught, not thrown)
-      expect(result.success).toBe(false);
-      expect(result.error).toBe('Index creation failed');
-      // initializeScheduler should NOT have been called since the error occurred before it
-      expect(mockInitializeScheduler).not.toHaveBeenCalled();
-    });
-
-    it('should create indexes before initializing the scheduler', async () => {
-      const callOrder = [];
-      mockCreateScraperIndexes.mockImplementation(async () => {
-        callOrder.push('createScraperIndexes');
-      });
-      mockInitializeScheduler.mockImplementation(() => {
-        callOrder.push('initializeScheduler');
-      });
-
-      const config = { SCRAPER_ENABLED: true };
-      await runScraperInit(db, config);
-
-      expect(callOrder).toEqual(['createScraperIndexes', 'initializeScheduler']);
-    });
-
-    it('should not initialize scheduler when SCRAPER_ENABLED is false', async () => {
-      const config = { SCRAPER_ENABLED: false };
-
-      const result = await runScraperInit(db, config);
-
-      expect(result.success).toBe(true);
-      expect(mockCreateScraperIndexes).toHaveBeenCalledTimes(1);
-      expect(mockInitializeScheduler).not.toHaveBeenCalled();
-    });
-  });
-
-  describe('server.js imports and exports verification', () => {
-    it('should be able to import createScraperIndexes from scraperService', () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      expect(typeof createScraperIndexes).toBe('function');
-    });
-
-    it('should be able to import initializeScheduler from scraperJob', () => {
-      const { initializeScheduler } = require('../../jobs/scraperJob');
-      expect(typeof initializeScheduler).toBe('function');
-    });
-
-    it('should be able to import registerSignalHandlers from scraperJob', () => {
-      const { registerSignalHandlers } = require('../../jobs/scraperJob');
-      expect(typeof registerSignalHandlers).toBe('function');
-    });
-
-    it('should be able to import SCRAPER_ENABLED from scraper config', () => {
-      const scraperConfig = require('../../config/scraper');
-      expect(typeof scraperConfig.SCRAPER_ENABLED).toBe('boolean');
-    });
-  });
-
-  describe('createScraperIndexes error handling with real db', () => {
-    it('should handle index creation gracefully with a valid db', async () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      const { createLogger } = require('../../services/scraperLogger');
-      const logger = createLogger('test-server-init');
-
-      // Should not throw - creates indexes on the test db
-      await expect(createScraperIndexes(db, logger)).resolves.not.toThrow();
-    });
-
-    it('should throw when given an invalid db object', async () => {
-      const { createScraperIndexes } = require('../../services/scraperService');
-      const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() };
-
-      // Passing null should throw (simulates what the try/catch in server.js protects against)
-      await expect(createScraperIndexes(null, mockLogger)).rejects.toThrow();
-    });
-  });
-
-  describe('registerSignalHandlers', () => {
-    it('should register SIGTERM and SIGINT handlers without throwing', () => {
-      const { registerSignalHandlers } = require('../../jobs/scraperJob');
-
-      // Should not throw when called
-      expect(() => registerSignalHandlers()).not.toThrow();
-    });
-  });
-});

__tests__/scraper/upsertUnits.test.js

@@ -56,7 +56,7 @@ beforeEach(async () => {
 });
 
 describe('upsertUnits', () => {
-  const UNITS_COLLECTION = 'units_scraper';
+  const UNITS_COLLECTION = 'units_migration_test';
 
   // ---------------------------------------------------------------
   // 1. bulkWrite is called with correct updateOne operations

config/scraper.js

@@ -13,7 +13,7 @@ module.exports = {
 
   // Scheduling configuration
   SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
-  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
+  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
   SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
 
   // HTTP settings
@@ -30,11 +30,10 @@ module.exports = {
   // Graceful shutdown timeout (how long to wait for running job before force-stopping)
   SHUTDOWN_TIMEOUT: parseInt(process.env.SCRAPER_SHUTDOWN_TIMEOUT, 10) || 30000,
 
-  // MongoDB collection names - defaults to validation collections for scraper validation;
+  // MongoDB collection names (environment variable overrides for development isolation)
-  // switch to production collections via env vars (SCRAPER_UNITS_COLLECTION, SCRAPER_PRICES_COLLECTION) when ready
   COLLECTIONS: {
-    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_scraper',
+    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
-    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_scraper',
+    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
     DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
     SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
   }

server.js

@@ -11,9 +11,6 @@ const activityRoutes = require('./routes/activity');
 const adminRoutes = require('./routes/admin');
 const { createIndexes } = require('./models/user');
 const { createActivityIndexes } = require('./services/activityLogger');
-const { createScraperIndexes } = require('./services/scraperService');
-const { initializeScheduler, registerSignalHandlers } = require('./jobs/scraperJob');
-const scraperConfig = require('./config/scraper');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
@@ -83,32 +80,14 @@ async function connectToMongoDB() {
     await createIndexes(db);
     await createActivityIndexes(db);
     console.log('✅ Passport configured and indexes created');
-
-    // Initialize scraper (indexes + scheduler) - errors logged but don't crash server
-    try {
-      const { createLogger } = require('./services/scraperLogger');
-      const scraperLogger = createLogger('server-init');
-      await createScraperIndexes(db, scraperLogger);
-      if (scraperConfig.SCRAPER_ENABLED) {
-        initializeScheduler(db);
-        console.log('✅ Scraper scheduler initialized');
-      } else {
-        console.log('ℹ️ Scraper scheduling disabled');
-      }
-    } catch (error) {
-      console.error('⚠️ Scraper initialization failed (server continues):', error.message);
-    }
   } catch (error) {
     console.error('❌ Failed to connect to MongoDB:', error);
     process.exit(1);
   }
 }
 
-// Register scraper signal handlers for graceful shutdown
+// Helper function to get today's date in UTC
-registerSignalHandlers();
+const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
-
-// Helper function to get today's date in configured timezone
-const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
 
 // Cache for the most recent date with data
 let cachedLatestDate = null;
@@ -143,7 +122,7 @@ const getLatestDateWithData = async () => {
   }
 
   // Fallback to UTC today if no data found
-  return getTodayDate();
+  return getTodayDateUTC();
 };
 
 // Helper function to get yesterday relative to the latest date with data
@@ -641,10 +620,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
     ]).toArray();
 
     // 4. Historical Comparison - current vs 30/90/365 day averages
-    const tz = scraperConfig.SCRAPER_TIMEZONE;
+    const now = new Date();
-    const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
-    const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
-    const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
+    const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
 
     const [currentAvg, avg30, avg90, avg365] = await Promise.all([
       db.collection(PRICES_COLLECTION).aggregate([

services/scraperService.js

@@ -626,11 +626,11 @@ async function updateDailySummary(db, summaryData, logger) {
 // ============================================================
 
 /**
- * Get today's date in YYYY-MM-DD format using the configured timezone.
+ * Get today's date in YYYY-MM-DD format (UTC).
  * @returns {string} Today's date string
  */
-function getToday() {
+function getTodayUTC() {
-  return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
+  return new Date().toISOString().split('T')[0];
 }
 
 /**
@@ -730,37 +730,6 @@ async function recordScraperRun(db, runData, logger) {
   }
 }
 
-// ============================================================
-// Index Management
-// ============================================================
-
-/**
- * Create indexes on the scraper_runs collection.
- * Should be called once during application startup.
- * Idempotent - safe to call multiple times.
- *
- * @param {Db} db - MongoDB database instance
- * @param {Object} logger - Logger instance
- * @returns {Promise<void>}
- */
-async function createScraperIndexes(db, logger) {
-  const collection = db.collection(config.COLLECTIONS.SCRAPER_RUNS);
-
-  // Compound index for querying runs by status sorted by most recent
-  await collection.createIndex(
-    { status: 1, startedAt: -1 },
-    { name: 'status_startedAt' }
-  );
-
-  // Index for sorting all runs by start time (most recent first)
-  await collection.createIndex(
-    { startedAt: -1 },
-    { name: 'startedAt_desc' }
-  );
-
-  logger.info('Scraper indexes created successfully');
-}
-
 // ============================================================
 // Main Orchestration Function
 // ============================================================
@@ -820,7 +789,7 @@ async function runScrape(db, options = {}) {
     result.unitsProcessed = units.length;
 
     // Step 4: Database operations
-    const today = getToday();
+    const today = getTodayUTC();
 
     // Get yesterday's unit codes for comparison
     const yesterdayUnits = await getYesterdayUnitCodes(db, today);
@@ -902,9 +871,8 @@ module.exports = {
   markStaleUnits,
   updateDailySummary,
   recordScraperRun,
-  createScraperIndexes,
   // Export helpers for testing
-  getToday,
+  getTodayUTC,
   getYesterdayUnitCodes,
   getYesterday,
   parseInteger,