stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

SCRAPE-3: Implement fetchPage() with axios, timeout, retry (#8)

Browse Source 
Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>
This commit is contained in:
Stephen Minakian
2026-02-05 15:58:07 -07:00
committed by stephen
parent 2993d019c5
commit c6beaf8333
3 changed files with 651 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
config/scraper.js
View File

@ -17,14 +17,14 @@ module.exports = {
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT) || 30000,
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000,
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
// Retry configuration for HTTP requests
RETRY_CONFIG: {
maxRetries: 3,
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
timeout: parseInt(process.env.SCRAPER_TIMEOUT) || 30000
timeout: parseInt(process.env.SCRAPER_TIMEOUT, 10) || 30000
},
// MongoDB collection names (environment variable overrides for development isolation)