SCRAPE-2: Add scraper config constants #7
192
__tests__/scraper/scraperConfig.test.js
Normal file
192
__tests__/scraper/scraperConfig.test.js
Normal file
@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Tests for config/scraper.js
|
||||
* Tests scraper configuration constants and environment variable overrides
|
||||
*/
|
||||
|
||||
describe('config/scraper', () => {
|
||||
// Store original env values to restore after tests
|
||||
const originalEnv = { ...process.env };
|
||||
|
||||
beforeEach(() => {
|
||||
// Clear all scraper-related env vars before each test
|
||||
delete process.env.SCRAPER_SCHEDULE;
|
||||
delete process.env.SCRAPER_TIMEZONE;
|
||||
delete process.env.SCRAPER_ENABLED;
|
||||
delete process.env.SCRAPER_TIMEOUT;
|
||||
delete process.env.SCRAPER_UNITS_COLLECTION;
|
||||
delete process.env.SCRAPER_PRICES_COLLECTION;
|
||||
delete process.env.SCRAPER_SUMMARIES_COLLECTION;
|
||||
delete process.env.SCRAPER_RUNS_COLLECTION;
|
||||
|
||||
// Clear the require cache to reload config with fresh env
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
// Restore original environment
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
describe('TARGET_URL', () => {
|
||||
test('should be set to the correct apartment listing URL', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.TARGET_URL).toBe(
|
||||
'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_SCHEDULE', () => {
|
||||
test('should default to "0 6 * * *" when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_SCHEDULE).toBe('0 6 * * *');
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_SCHEDULE = '0 12 * * *';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_SCHEDULE).toBe('0 12 * * *');
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_TIMEZONE', () => {
|
||||
test('should default to "UTC" when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEZONE).toBe('UTC');
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_TIMEZONE = 'America/Denver';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_ENABLED', () => {
|
||||
test('should default to true when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
|
||||
test('should be false when env var is set to "false"', () => {
|
||||
process.env.SCRAPER_ENABLED = 'false';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(false);
|
||||
});
|
||||
|
||||
test('should be true when env var is set to any other value', () => {
|
||||
process.env.SCRAPER_ENABLED = 'true';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
|
||||
test('should be true when env var is empty string', () => {
|
||||
process.env.SCRAPER_ENABLED = '';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_ENABLED).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SCRAPER_TIMEOUT', () => {
|
||||
test('should default to 30000 when env var not set', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(30000);
|
||||
});
|
||||
|
||||
test('should use env var override when set', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '60000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(60000);
|
||||
});
|
||||
|
||||
test('should parse string to integer', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '45000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(typeof config.SCRAPER_TIMEOUT).toBe('number');
|
||||
expect(config.SCRAPER_TIMEOUT).toBe(45000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('USER_AGENT', () => {
|
||||
test('should be a valid User-Agent string', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.USER_AGENT).toBe('Mozilla/5.0 (compatible; ApartmentScraper/1.0)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('RETRY_CONFIG', () => {
|
||||
test('should have maxRetries property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('maxRetries');
|
||||
expect(config.RETRY_CONFIG.maxRetries).toBe(3);
|
||||
});
|
||||
|
||||
test('should have baseDelay property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('baseDelay');
|
||||
expect(config.RETRY_CONFIG.baseDelay).toBe(1000);
|
||||
});
|
||||
|
||||
test('should have timeout property', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG).toHaveProperty('timeout');
|
||||
expect(config.RETRY_CONFIG.timeout).toBe(30000);
|
||||
});
|
||||
|
||||
test('timeout should respect SCRAPER_TIMEOUT env var', () => {
|
||||
process.env.SCRAPER_TIMEOUT = '45000';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.RETRY_CONFIG.timeout).toBe(45000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('COLLECTIONS', () => {
|
||||
describe('default values', () => {
|
||||
test('UNITS should default to "units_migration_test"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.UNITS).toBe('units_migration_test');
|
||||
});
|
||||
|
||||
test('PRICES should default to "unit_prices_migration_test"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.PRICES).toBe('unit_prices_migration_test');
|
||||
});
|
||||
|
||||
test('DAILY_SUMMARIES should default to "daily_summaries"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('daily_summaries');
|
||||
});
|
||||
|
||||
test('SCRAPER_RUNS should default to "scraper_runs"', () => {
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('scraper_runs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('env var overrides', () => {
|
||||
test('UNITS should respect SCRAPER_UNITS_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_UNITS_COLLECTION = 'units_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.UNITS).toBe('units_custom');
|
||||
});
|
||||
|
||||
test('PRICES should respect SCRAPER_PRICES_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_PRICES_COLLECTION = 'prices_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.PRICES).toBe('prices_custom');
|
||||
});
|
||||
|
||||
test('DAILY_SUMMARIES should respect SCRAPER_SUMMARIES_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_SUMMARIES_COLLECTION = 'summaries_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.DAILY_SUMMARIES).toBe('summaries_custom');
|
||||
});
|
||||
|
||||
test('SCRAPER_RUNS should respect SCRAPER_RUNS_COLLECTION env var', () => {
|
||||
process.env.SCRAPER_RUNS_COLLECTION = 'runs_custom';
|
||||
const config = require('../../config/scraper');
|
||||
expect(config.COLLECTIONS.SCRAPER_RUNS).toBe('runs_custom');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
37
config/scraper.js
Normal file
37
config/scraper.js
Normal file
@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Scraper Configuration Module
|
||||
*
|
||||
* Contains all constants and settings for the apartment price scraper.
|
||||
* All values can be overridden via environment variables for deployment flexibility.
|
||||
*
|
||||
* See py_migration/phase-3/3.2-TECHNICAL-DESIGN.md Section 5.3 for full documentation.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
// Target URL for apartment listings
|
||||
TARGET_URL: 'https://countryclubtowersandgardens.com/property/country-club-towers/apartments/?spaces_tab=unit',
|
||||
|
||||
// Scheduling configuration
|
||||
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
|
||||
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
|
||||
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
|
||||
|
||||
// HTTP settings
|
||||
SCRAPER_TIMEOUT: parseInt(process.env.SCRAPER_TIMEOUT) || 30000,
|
||||
USER_AGENT: 'Mozilla/5.0 (compatible; ApartmentScraper/1.0)',
|
||||
|
||||
// Retry configuration for HTTP requests
|
||||
RETRY_CONFIG: {
|
||||
maxRetries: 3,
|
||||
baseDelay: 1000, // 1 second, exponential backoff: 1s, 2s, 4s
|
||||
timeout: parseInt(process.env.SCRAPER_TIMEOUT) || 30000
|
||||
},
|
||||
|
||||
// MongoDB collection names (environment variable overrides for development isolation)
|
||||
COLLECTIONS: {
|
||||
UNITS: process.env.SCRAPER_UNITS_COLLECTION || 'units_migration_test',
|
||||
PRICES: process.env.SCRAPER_PRICES_COLLECTION || 'unit_prices_migration_test',
|
||||
DAILY_SUMMARIES: process.env.SCRAPER_SUMMARIES_COLLECTION || 'daily_summaries',
|
||||
SCRAPER_RUNS: process.env.SCRAPER_RUNS_COLLECTION || 'scraper_runs'
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user