SCRAPE-2: Add scraper config constants (#7)

This commit is contained in:
2026-01-31 20:54:43 -07:00
parent 3af5a90c93
commit 2993d019c5
3 changed files with 230 additions and 2 deletions

37
config/scraper.js Normal file
View File

@ -0,0 +1,37 @@
/**
* Scraper Configuration Module
*
* Contains all constants and settings for the apartment price scraper.
* All values can be overridden via environment variables for deployment flexibility.
*
* See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
*/
module.exports = {
// Target URL for apartment listings
TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
// Scheduling configuration
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT) || 30000,
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
// Retry configuration for HTTP requests
RETRY_CONFIG: {
maxRetries: 3,
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
timeout: parseInt(process.env.SCRAPER_TIMEOUT) || 30000
},
// MongoDB collection names (environment variable overrides for development isolation)
COLLECTIONS: {
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
}
};