SCRAPE-3: Implement fetchPage() with axios, timeout, retry (#8)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

__tests__/scraper/fetchPage.test.js

@@ -0,0 +1,535 @@
+/**
+ * Tests for fetchPage function
+ *
+ * Acceptance Criteria:
+ * - Uses axios for HTTP GET
+ * - Enforces timeout from config (default 30s)
+ * - Sets User-Agent header from config
+ * - Implements retry with exponential backoff (1s, 2s, 4s)
+ * - Retries on 5xx errors and network timeouts
+ * - Does NOT retry on 4xx errors
+ * - Logs each attempt with attempt number
+ * - Returns HTML string on success
+ * - Throws error after all retries exhausted
+ */
+
+const axios = require('axios');
+
+// Mock axios
+jest.mock('axios');
+
+// Mock config
+jest.mock('../../config/scraper', () => ({
+  RETRY_CONFIG: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    timeout: 30000
+  },
+  USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)'
+}));
+
+// Import after mocking
+const { fetchPage } = require('../../services/scraperService');
+const config = require('../../config/scraper');
+
+describe('fetchPage', () => {
+  let mockLogger;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+
+    // Create mock logger
+    mockLogger = {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn()
+    };
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('successful requests', () => {
+    it('should return HTML string on success', async () => {
+      const htmlContent = '<html><body>Test content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      const result = await fetchPage('https://example.com', mockLogger);
+
+      expect(result).toBe(htmlContent);
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use axios for HTTP GET', async () => {
+      const url = 'https://example.com/apartments';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage(url, mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(url, expect.any(Object));
+    });
+
+    it('should set timeout from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          timeout: config.RETRY_CONFIG.timeout
+        })
+      );
+    });
+
+    it('should set User-Agent header from config', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'User-Agent': config.USER_AGENT
+          })
+        })
+      );
+    });
+
+    it('should log the attempt with attempt number', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({
+          url: 'https://example.com',
+          attempt: 1
+        })
+      );
+    });
+
+    it('should log successful fetch with status and content length', async () => {
+      const htmlContent = '<html><body>Content</body></html>';
+
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: htmlContent
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Page fetched successfully',
+        expect.objectContaining({
+          status: 200,
+          contentLength: htmlContent.length
+        })
+      );
+    });
+  });
+
+  describe('retry behavior on 5xx errors', () => {
+    it('should retry on 500 Internal Server Error', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // Fast-forward through the retry delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 502 Bad Gateway', async () => {
+      const error502 = new Error('Bad Gateway');
+      error502.response = { status: 502 };
+
+      axios.get
+        .mockRejectedValueOnce(error502)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 503 Service Unavailable', async () => {
+      const error503 = new Error('Service Unavailable');
+      error503.response = { status: 503 };
+
+      axios.get
+        .mockRejectedValueOnce(error503)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on 504 Gateway Timeout', async () => {
+      const error504 = new Error('Gateway Timeout');
+      error504.response = { status: 504 };
+
+      axios.get
+        .mockRejectedValueOnce(error504)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('retry behavior on network errors', () => {
+    it('should retry on network timeout (ECONNABORTED)', async () => {
+      const timeoutError = new Error('timeout of 30000ms exceeded');
+      timeoutError.code = 'ECONNABORTED';
+      // Network errors don't have a response property
+
+      axios.get
+        .mockRejectedValueOnce(timeoutError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      const result = await promise;
+
+      expect(result).toBe('<html></html>');
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on DNS resolution failure (ENOTFOUND)', async () => {
+      const dnsError = new Error('getaddrinfo ENOTFOUND');
+      dnsError.code = 'ENOTFOUND';
+
+      axios.get
+        .mockRejectedValueOnce(dnsError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+
+    it('should retry on connection refused (ECONNREFUSED)', async () => {
+      const connError = new Error('connect ECONNREFUSED');
+      connError.code = 'ECONNREFUSED';
+
+      axios.get
+        .mockRejectedValueOnce(connError)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('no retry on 4xx errors', () => {
+    it('should NOT retry on 400 Bad Request', async () => {
+      const error400 = new Error('Bad Request');
+      error400.response = { status: 400 };
+
+      axios.get.mockRejectedValueOnce(error400);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 401 Unauthorized', async () => {
+      const error401 = new Error('Unauthorized');
+      error401.response = { status: 401 };
+
+      axios.get.mockRejectedValueOnce(error401);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 403 Forbidden', async () => {
+      const error403 = new Error('Forbidden');
+      error403.response = { status: 403 };
+
+      axios.get.mockRejectedValueOnce(error403);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 404 Not Found', async () => {
+      const error404 = new Error('Not Found');
+      error404.response = { status: 404 };
+
+      axios.get.mockRejectedValueOnce(error404);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+
+    it('should NOT retry on 429 Too Many Requests', async () => {
+      const error429 = new Error('Too Many Requests');
+      error429.response = { status: 429 };
+
+      axios.get.mockRejectedValueOnce(error429);
+
+      await expect(fetchPage('https://example.com', mockLogger))
+        .rejects.toThrow();
+
+      expect(axios.get).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('exponential backoff', () => {
+    it('should use exponential backoff delays: 1s, 2s, 4s', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      // Fail 3 times, then succeed
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      // First retry: 1 second delay
+      await jest.advanceTimersByTimeAsync(1000);
+
+      // Second retry: 2 second delay
+      await jest.advanceTimersByTimeAsync(2000);
+
+      // Third retry: 4 second delay
+      await jest.advanceTimersByTimeAsync(4000);
+
+      await promise;
+
+      expect(axios.get).toHaveBeenCalledTimes(4);
+    });
+
+    it('should log each retry attempt with attempt number', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+
+      await jest.advanceTimersByTimeAsync(1000);
+      await jest.advanceTimersByTimeAsync(2000);
+
+      await promise;
+
+      // Check that attempt numbers were logged
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 1 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 2 })
+      );
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Fetching page',
+        expect.objectContaining({ attempt: 3 })
+      );
+    });
+
+    it('should log failed attempts with error details', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Fetch attempt failed',
+        expect.objectContaining({
+          attempt: 1,
+          statusCode: 500,
+          isRetryable: true
+        })
+      );
+    });
+
+    it('should log wait time before retry', async () => {
+      const error500 = new Error('Internal Server Error');
+      error500.response = { status: 500 };
+
+      axios.get
+        .mockRejectedValueOnce(error500)
+        .mockResolvedValueOnce({ status: 200, data: '<html></html>' });
+
+      const promise = fetchPage('https://example.com', mockLogger);
+      await jest.advanceTimersByTimeAsync(1000);
+
+      await promise;
+
+      expect(mockLogger.info).toHaveBeenCalledWith(
+        'Waiting before retry',
+        expect.objectContaining({ delay: 1000 })
+      );
+    });
+  });
+
+  describe('retry exhaustion', () => {
+    // Use real timers for exhaustion tests to avoid promise handling issues
+    beforeEach(() => {
+      jest.useRealTimers();
+    });
+
+    afterEach(() => {
+      jest.useFakeTimers();
+    });
+
+    it('should throw error after all retries exhausted (4 attempts total)', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Internal Server Error');
+        error500.response = { status: 500 };
+
+        // All 4 attempts fail (1 initial + 3 retries)
+        axios.get
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500)
+          .mockRejectedValueOnce(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toThrow('Internal Server Error');
+
+        expect(axios.get).toHaveBeenCalledTimes(4);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+
+    it('should throw the last error when retries are exhausted', async () => {
+      // Override config for faster tests
+      const originalBaseDelay = config.RETRY_CONFIG.baseDelay;
+      config.RETRY_CONFIG.baseDelay = 1; // 1ms instead of 1000ms
+
+      try {
+        const error500 = new Error('Server Error');
+        error500.response = { status: 500 };
+
+        axios.get.mockRejectedValue(error500);
+
+        await expect(fetchPage('https://example.com', mockLogger))
+          .rejects.toBe(error500);
+      } finally {
+        config.RETRY_CONFIG.baseDelay = originalBaseDelay;
+      }
+    });
+  });
+
+  describe('axios configuration', () => {
+    it('should set maxRedirects to 5', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      expect(axios.get).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          maxRedirects: 5
+        })
+      );
+    });
+
+    it('should accept 2xx and 3xx status codes via validateStatus', async () => {
+      axios.get.mockResolvedValueOnce({
+        status: 200,
+        data: '<html></html>'
+      });
+
+      await fetchPage('https://example.com', mockLogger);
+
+      const callArgs = axios.get.mock.calls[0][1];
+      const validateStatus = callArgs.validateStatus;
+
+      // Should accept 2xx
+      expect(validateStatus(200)).toBe(true);
+      expect(validateStatus(201)).toBe(true);
+      expect(validateStatus(204)).toBe(true);
+
+      // Should accept 3xx
+      expect(validateStatus(301)).toBe(true);
+      expect(validateStatus(302)).toBe(true);
+
+      // Should reject 4xx and 5xx
+      expect(validateStatus(400)).toBe(false);
+      expect(validateStatus(404)).toBe(false);
+      expect(validateStatus(500)).toBe(false);
+    });
+  });
+});

config/scraper.js

@@ -17,14 +17,14 @@ module.exports = {
   SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
 
   // HTTP settings
-  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT) || 30000,
+  SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
   USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
 
   // Retry configuration for HTTP requests
   RETRY_CONFIG: {
     maxRetries: 3,
     baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
-    timeout: parseInt(process.env.SCRAPER_TIMEOUT) || 30000
+    timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
   },
 
   // MongoDB collection names (environment variable overrides for development isolation)

services/scraperService.js

@@ -0,0 +1,114 @@
+/**
+ * Scraper Service
+ *
+ * Core scraper logic including HTTP fetching, HTML parsing,
+ * data transformation, and database operations.
+ */
+
+const axios = require('axios');
+const config = require('../config/scraper');
+
+/**
+ * Sleep utility for retry delays
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Determine if an error is retryable
+ * @param {Error} error - Axios error
+ * @returns {boolean} True if should retry
+ */
+function isRetryableError(error) {
+  // Network errors (timeout, DNS, connection) don't have a response property
+  if (!error.response) {
+    return true;
+  }
+
+  const status = error.response.status;
+
+  // 5xx server errors are retryable
+  if (status >= 500) {
+    return true;
+  }
+
+  // 429 Too Many Requests - do not retry immediately
+  if (status === 429) {
+    return false;
+  }
+
+  // 4xx client errors - do not retry
+  return false;
+}
+
+/**
+ * Fetch page HTML with retry logic
+ * @param {string} url - Target URL
+ * @param {Object} logger - Logger instance
+ * @returns {Promise<string>} HTML content
+ * @throws {Error} After all retries exhausted
+ */
+async function fetchPage(url, logger) {
+  const { maxRetries, baseDelay, timeout } = config.RETRY_CONFIG;
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    try {
+      logger.info('Fetching page', { url, attempt });
+
+      const response = await axios.get(url, {
+        timeout,
+        headers: {
+          'User-Agent': config.USER_AGENT
+        },
+        maxRedirects: 5,
+        validateStatus: (status) => status < 400 // Accept 2xx and 3xx
+      });
+
+      logger.info('Page fetched successfully', {
+        status: response.status,
+        contentLength: response.data.length
+      });
+
+      return response.data;
+
+    } catch (error) {
+      lastError = error;
+
+      // Determine if error is retryable
+      const isRetryable = isRetryableError(error);
+
+      logger.warn('Fetch attempt failed', {
+        attempt,
+        errorType: error.name,
+        errorMessage: error.message,
+        statusCode: error.response?.status,
+        isRetryable
+      });
+
+      // Don't retry non-retryable errors (4xx)
+      if (!isRetryable) {
+        throw error;
+      }
+
+      // Don't wait after last attempt
+      if (attempt <= maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+        logger.info('Waiting before retry', { delay });
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+module.exports = {
+  fetchPage,
+  // Export for testing
+  isRetryableError,
+  sleep
+};