SCRAPE-2: Add scraper config constants (#7)
This commit is contained in:
37
config/scraper.js
Normal file
37
config/scraper.js
Normal file
@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Scraper Configuration Module
|
||||
*
|
||||
* Contains all constants and settings for the apartment price scraper.
|
||||
* All values can be overridden via environment variables for deployment flexibility.
|
||||
*
|
||||
* See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
// Target URL for apartment listings
|
||||
TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
|
||||
|
||||
// Scheduling configuration
|
||||
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
|
||||
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
|
||||
|
||||
// HTTP settings
|
||||
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT) || 30000,
|
||||
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
|
||||
|
||||
// Retry configuration for HTTP requests
|
||||
RETRY_CONFIG: {
|
||||
maxRetries: 3,
|
||||
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
|
||||
timeout: parseInt(process.env.SCRAPER_TIMEOUT) || 30000
|
||||
},
|
||||
|
||||
// MongoDB collection names (environment variable overrides for development isolation)
|
||||
COLLECTIONS: {
|
||||
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
|
||||
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
|
||||
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
|
||||
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user