Add fetchPage() with axios, timeout, and retry logic

Implement fetchPage() function for HTTP scraping:
- Uses axios for HTTP GET requests
- Configurable timeout (default 30s) and User-Agent header
- Retry with exponential backoff (1s, 2s, 4s) on 5xx and network errors
- Does not retry on 4xx client errors
- Logs each attempt with attempt number and error details
- Returns HTML string on success, throws after retries exhausted

.github/workflows/deploy.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
   workflow_dispatch:
 
 env:
@@ -12,6 +11,42 @@ env:
   IMAGE_NAME: apartment-api
 
 jobs:
+  # ============================================================
+  # Lint Job - ESLint static analysis
+  # ============================================================
+  lint:
+    name: Run Linting
+    runs-on: ubuntu-latest
+
+    outputs:
+      status: ${{ steps.lint.outcome }}
+      output: ${{ steps.lint.outputs.lint_output }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run ESLint
+        id: lint
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm run lint 2>&1)
+          EXIT_CODE=$?
+          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+
   # ============================================================
   # Test Job - Runs first, blocks everything if tests fail
   # ============================================================
@@ -19,6 +54,10 @@ jobs:
     name: Run Tests
     runs-on: ubuntu-latest
 
+    outputs:
+      status: ${{ steps.test.outcome }}
+      output: ${{ steps.test.outputs.test_output }}
+
     services:
       mongodb:
         image: mongo:7
@@ -44,12 +83,109 @@ jobs:
         run: npm ci
 
       - name: Run tests
-        run: npm test -- --runInBand
+        id: test
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm test -- --runInBand 2>&1)
+          EXIT_CODE=$?
+          echo "test_output<<EOF" >> $GITHUB_OUTPUT
+          echo "$OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
         env:
           MONGO_URI: mongodb://localhost:27018
           JWT_SECRET: test-jwt-secret-for-ci
           NODE_ENV: test
 
+  # ============================================================
+  # Notify Job - Send results to n8n webhook
+  # ============================================================
+  notify:
+    name: Send Webhook Notification
+    runs-on: ubuntu-latest
+    needs: [lint, test]
+    if: always()
+
+    steps:
+      - name: Send results to n8n webhook
+        run: |
+          # Determine overall status
+          LINT_STATUS="${{ needs.lint.result }}"
+          TEST_STATUS="${{ needs.test.result }}"
+
+          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
+            OVERALL_STATUS="success"
+          else
+            OVERALL_STATUS="failure"
+          fi
+
+          # Truncate outputs if too long (max 10000 chars each)
+          LINT_OUTPUT=$(echo '${{ needs.lint.outputs.output }}' | head -c 10000)
+          TEST_OUTPUT=$(echo '${{ needs.test.outputs.output }}' | head -c 10000)
+
+          # Build JSON payload
+          PAYLOAD=$(jq -n \
+            --arg repo "${{ github.repository }}" \
+            --arg branch "${{ github.head_ref || github.ref_name }}" \
+            --arg commit "${{ github.sha }}" \
+            --arg commit_short "$(echo '${{ github.sha }}' | cut -c1-7)" \
+            --arg commit_message "${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}" \
+            --arg author "${{ github.actor }}" \
+            --arg event "${{ github.event_name }}" \
+            --arg pr_number "${{ github.event.pull_request.number || '' }}" \
+            --arg run_id "${{ github.run_id }}" \
+            --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg overall_status "$OVERALL_STATUS" \
+            --arg lint_status "$LINT_STATUS" \
+            --arg lint_output "$LINT_OUTPUT" \
+            --arg test_status "$TEST_STATUS" \
+            --arg test_output "$TEST_OUTPUT" \
+            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            '{
+              repository: $repo,
+              branch: $branch,
+              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
+              commit: {
+                sha: $commit,
+                short_sha: $commit_short,
+                message: $commit_message,
+                author: $author
+              },
+              event: $event,
+              run: {
+                id: $run_id,
+                url: $run_url
+              },
+              status: $overall_status,
+              results: {
+                lint: {
+                  status: $lint_status,
+                  output: (if $lint_status != "success" then $lint_output else null end)
+                },
+                test: {
+                  status: $test_status,
+                  output: (if $test_status != "success" then $test_output else null end)
+                }
+              },
+              timestamp: $timestamp
+            }')
+
+          # Send webhook
+          curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "${{ secrets.N8N_WEBHOOK_URL }}" \
+            --fail --silent --show-error
+
+      - name: Fail if lint or tests failed
+        if: needs.lint.result != 'success' || needs.test.result != 'success'
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "   Lint: ${{ needs.lint.result }}"
+          echo "   Test: ${{ needs.test.result }}"
+          exit 1
+
   # ============================================================
   # Dependency Scan Job - Runs in parallel with tests
   # ============================================================
@@ -80,7 +216,7 @@ jobs:
   build:
     name: Build & Push Image
     runs-on: ubuntu-latest
-    needs: [test, scan-deps]
+    needs: [lint, test, scan-deps, notify]
     if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
 
     outputs:

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

__tests__/scraper/scraperConfig.test.js

@@ -0,0 +1,192 @@
+/**
+ * Tests for config/scraper.js
+ * Tests scraper configuration constants and environment variable overrides
+ */
+
+describe('config/scraper', () => {
+  // Store original env values to restore after tests
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    // Clear all scraper-related env vars before each test
+    delete process.env.SCRAPER_SCHEDULE;
+    delete process.env.SCRAPER_TIMEZONE;
+    delete process.env.SCRAPER_ENABLED;
+    delete process.env.SCRAPER_TIMEOUT;
+    delete process.env.SCRAPER_UNITS_COLLECTION;
+    delete process.env.SCRAPER_PRICES_COLLECTION;
+    delete process.env.SCRAPER_SUMMARIES_COLLECTION;
+    delete process.env.SCRAPER_RUNS_COLLECTION;
+
+    // Clear the require cache to reload config with fresh env
+    jest.resetModules();
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    process.env = originalEnv;
+  });
+
+  describe('TARGET_URL', () => {
+    test('should be set to the correct apartment listing URL', () => {
+      const config = require('../../config/scraper');
+      expect(config.TARGET_URL).toBe(
+        'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
+      );
+    });
+  });
+
+  describe('SCRAPER_SCHEDULE', () => {
+    test('should default to "0 6 * * *" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_SCHEDULE = '0 12 * * *';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
+    });
+  });
+
+  describe('SCRAPER_TIMEZONE', () => {
+    test('should default to "UTC" when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('UTC');
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEZONE = 'America/Denver';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
+    });
+  });
+
+  describe('SCRAPER_ENABLED', () => {
+    test('should default to true when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be false when env var is set to "false"', () => {
+      process.env.SCRAPER_ENABLED = 'false';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(false);
+    });
+
+    test('should be true when env var is set to any other value', () => {
+      process.env.SCRAPER_ENABLED = 'true';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+
+    test('should be true when env var is empty string', () => {
+      process.env.SCRAPER_ENABLED = '';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_ENABLED).toBe(true);
+    });
+  });
+
+  describe('SCRAPER_TIMEOUT', () => {
+    test('should default to 30000 when env var not set', () => {
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(30000);
+    });
+
+    test('should use env var override when set', () => {
+      process.env.SCRAPER_TIMEOUT = '60000';
+      const config = require('../../config/scraper');
+      expect(config.SCRAPER_TIMEOUT).toBe(60000);
+    });
+
+    test('should parse string to integer', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
+      expect(config.SCRAPER_TIMEOUT).toBe(45000);
+    });
+  });
+
+  describe('USER_AGENT', () => {
+    test('should be a valid User-Agent string', () => {
+      const config = require('../../config/scraper');
+      expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
+    });
+  });
+
+  describe('RETRY_CONFIG', () => {
+    test('should have maxRetries property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
+      expect(config.RETRY_CONFIG.maxRetries).toBe(3);
+    });
+
+    test('should have baseDelay property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
+      expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
+    });
+
+    test('should have timeout property', () => {
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG).toHaveProperty('timeout');
+      expect(config.RETRY_CONFIG.timeout).toBe(30000);
+    });
+
+    test('timeout should respect SCRAPER_TIMEOUT env var', () => {
+      process.env.SCRAPER_TIMEOUT = '45000';
+      const config = require('../../config/scraper');
+      expect(config.RETRY_CONFIG.timeout).toBe(45000);
+    });
+  });
+
+  describe('COLLECTIONS', () => {
+    describe('default values', () => {
+      test('UNITS should default to "units_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
+      });
+
+      test('PRICES should default to "unit_prices_migration_test"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
+      });
+
+      test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
+      });
+
+      test('SCRAPER_RUNS should default to "scraper_runs"', () => {
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
+      });
+    });
+
+    describe('env var overrides', () => {
+      test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
+        process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.UNITS).toBe('units_custom');
+      });
+
+      test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
+        process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
+      });
+
+      test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
+        process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
+      });
+
+      test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
+        process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
+        const config = require('../../config/scraper');
+        expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
+      });
+    });
+  });
+});

__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

config/scraper.js

@@ -0,0 +1,37 @@
+/**
+ * Scraper Configuration Module
+ *
+ * Contains all constants and settings for the apartment price scraper.
+ * All values can be overridden via environment variables for deployment flexibility.
+ *
+ * See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
+ */
+
+module.exports = {
+  // Target URL for apartment listings
+  TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
+
+  // Scheduling configuration
+  SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
+  SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
+  SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
+
+  // HTTP settings
+  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
+
+  // Retry configuration for HTTP requests
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
+    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
+  },
+
+  // MongoDB collection names (environment variable overrides for development isolation)
+  COLLECTIONS: {
+    UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
+    PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
+    DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
+    SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
+  }
+};

eslint.config.mjs

@@ -0,0 +1,34 @@
+import js from '@eslint/js';
+import globals from 'globals';
+
+export default [
+  js.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'commonjs',
+      globals: {
+        ...globals.node,
+        ...globals.jest,
+      },
+    },
+    rules: {
+      // Allow unused vars with underscore prefix, and unused function args
+      'no-unused-vars': ['warn', { argsIgnorePattern: '^_|^err|^error', varsIgnorePattern: '^_', args: 'none' }],
+      'no-console': 'off',
+      // Formatting rules as warnings (won't fail CI, but will be reported)
+      'semi': ['warn', 'always'],
+      'no-multiple-empty-lines': ['warn', { max: 2 }],
+      'eol-last': ['warn', 'always'],
+      'no-trailing-spaces': 'warn',
+    },
+  },
+  {
+    ignores: [
+      'node_modules/**',
+      'coverage/**',
+      '*.min.js',
+      'eslint.config.mjs',
+    ],
+  },
+];

package.json

@@ -8,7 +8,9 @@
         "dev": "nodemon server.js",
         "test": "jest",
         "test:watch": "jest --watch",
-        "test:coverage": "jest --coverage"
+        "test:coverage": "jest --coverage",
+        "lint": "eslint .",
+        "lint:fix": "eslint . --fix"
     },
     "keywords": [
         "apartments",
@@ -19,17 +21,24 @@
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
+        "axios": "^1.13.4",
+        "cheerio": "^1.2.0",
         "cookie-parser": "^1.4.7",
         "cors": "^2.8.5",
+        "cron-parser": "^5.5.0",
         "express": "^4.18.2",
         "express-rate-limit": "^7.1.5",
         "jsonwebtoken": "^9.0.3",
         "mongodb": "^6.3.0",
+        "node-cron": "^4.2.1",
         "passport": "^0.7.0",
         "passport-google-oauth20": "^2.0.0",
         "uuid": "^13.0.0"
     },
     "devDependencies": {
+        "@eslint/js": "^9.18.0",
+        "eslint": "^9.18.0",
+        "globals": "^15.14.0",
         "jest": "^30.2.0",
         "mongodb-memory-server": "^11.0.1",
         "nodemon": "^3.0.2",

services/scraperLogger.js

@@ -0,0 +1,164 @@
+/**
+ * Scraper-specific structured JSON logger
+ * Produces one JSON object per line to stdout
+ */
+
+const LEVELS = {
+  info: 'info',
+  warn: 'warn',
+  error: 'error'
+};
+
+/**
+ * Create a logger instance scoped to a job ID
+ * @param {string} jobId - Job identifier for correlation
+ * @returns {Object} Logger object with info, warn, error methods
+ */
+function createLogger(jobId) {
+  /**
+   * Internal log function
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} context - Additional context data
+   */
+  const log = (level, message, context = {}) => {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      message: truncateMessage(message),
+      jobId,
+      context: sanitizeContext(context)
+    };
+
+    // Output as single-line JSON
+    console.log(JSON.stringify(entry));
+  };
+
+  return {
+    info: (message, context) => log(LEVELS.info, message, context),
+    warn: (message, context) => log(LEVELS.warn, message, context),
+    error: (message, context) => log(LEVELS.error, message, context)
+  };
+}
+
+/**
+ * Truncate message to prevent log bloat
+ * @param {string} message - Message to truncate
+ * @param {number} maxLength - Maximum length (default 1000)
+ * @returns {string} Truncated message
+ */
+function truncateMessage(message, maxLength = 1000) {
+  if (message === null || message === undefined) {
+    return '';
+  }
+  const str = String(message);
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.substring(0, maxLength) + '... [truncated]';
+}
+
+/**
+ * Recursively process an object to handle Buffers, circular references, and sensitive data
+ * @param {*} obj - Object to process
+ * @param {WeakSet} seen - Set of seen objects for circular reference detection
+ * @returns {*} Processed value
+ */
+function processValue(obj, seen = new WeakSet()) {
+  // Handle null/undefined
+  if (obj === null) {
+    return null;
+  }
+  if (obj === undefined) {
+    return null;
+  }
+
+  // Handle Buffer BEFORE checking for object (Buffer is an object)
+  if (Buffer.isBuffer(obj)) {
+    return `[Buffer: ${obj.length} bytes]`;
+  }
+
+  // Handle strings - check for MongoDB connection strings
+  if (typeof obj === 'string') {
+    if (/mongodb(\+srv)?:\/\//.test(obj)) {
+      return redactConnectionString(obj);
+    }
+    return obj;
+  }
+
+  // Handle primitives
+  if (typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Handle circular references
+  if (seen.has(obj)) {
+    return '[Circular]';
+  }
+  seen.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => processValue(item, seen));
+  }
+
+  // Handle plain objects
+  const result = {};
+  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
+
+  for (const key of Object.keys(obj)) {
+    // Check for sensitive keys
+    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
+      result[key] = '[REDACTED]';
+    } else {
+      result[key] = processValue(obj[key], seen);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sanitize context object for safe logging
+ * - Remove circular references
+ * - Redact sensitive data
+ * - Handle special types (Buffer, undefined)
+ * @param {Object} context - Context object
+ * @returns {Object} Sanitized context
+ */
+function sanitizeContext(context) {
+  if (!context || typeof context !== 'object') {
+    return {};
+  }
+
+  try {
+    return processValue(context);
+  } catch (error) {
+    // If sanitization fails, return empty context
+    return { sanitizationError: 'Failed to sanitize context' };
+  }
+}
+
+/**
+ * Redact credentials from MongoDB connection string
+ * @param {string} uri - Connection string
+ * @returns {string} Redacted string
+ */
+function redactConnectionString(uri) {
+  try {
+    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
+    return uri.replace(
+      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
+      'mongodb$1://[user]:[REDACTED]@'
+    );
+  } catch {
+    return '[REDACTED CONNECTION STRING]';
+  }
+}
+
+module.exports = {
+  createLogger,
+  LEVELS,
+  // Export internal functions for testing
+  redactConnectionString
+};

services/scraperService.js

@@ -0,0 +1,114 @@
+/**
+ * Scraper Service
+ *
+ * Core scraper logic including HTTP fetching, HTML parsing,
+ * data transformation, and database operations.
+ */
+
+const axios = require('axios');
+const config = require('../config/scraper');
+
+/**
+ * Sleep utility for retry delays
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Determine if an error is retryable
+ * @param {Error} error - Axios error
+ * @returns {boolean} True if should retry
+ */
+function isRetryableError(error) {
+  // Network errors (timeout, DNS, connection) don't have a response property
+  if (!error.response) {
+    return true;
+  }
+
+  const status = error.response.status;
+
+  // 5xx server errors are retryable
+  if (status >= 500) {
+    return true;
+  }
+
+  // 429 Too Many Requests - do not retry immediately
+  if (status === 429) {
+    return false;
+  }
+
+  // 4xx client errors - do not retry
+  return false;
+}
+
+/**
+ * Fetch page HTML with retry logic
+ * @param {string} url - Target URL
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<string>} HTML content
+ * @throws {Error} After all retries exhausted
+ */
+async function fetchPage(url, logger) {
+  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    try {
+      logger.info('Fetching page', { url, attempt });
+
+      const response = await axios.get(url, {
+        timeout,
+        headers: {
+          'User-Agent': config.USER_AGENT
+        },
+        maxRedirects: 5,
+        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
+      });
+
+      logger.info('Page fetched successfully', {
+        status: response.status,
+        contentLength: response.data.length
+      });
+
+      return response.data;
+
+    } catch (error) {
+      lastError = error;
+
+      // Determine if error is retryable
+      const isRetryable = isRetryableError(error);
+
+      logger.warn('Fetch attempt failed', {
+        attempt,
+        errorType: error.name,
+        errorMessage: error.message,
+        statusCode: error.response?.status,
+        isRetryable
+      });
+
+      // Don't retry non-retryable errors (4xx)
+      if (!isRetryable) {
+        throw error;
+      }
+
+      // Don't wait after last attempt
+      if (attempt <= maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+        logger.info('Waiting before retry', { delay });
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+module.exports = {
+  fetchPage,
+  // Export for testing
+  isRetryableError,
+  sleep
+};