stephen/apartment-dashboard-api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Releases Wiki Activity

Use Denver local time for scrape dates instead of UTC

Browse Source 
Change default SCRAPER_TIMEZONE from UTC to America/Denver so that
scrape date records (date_checked, daily summaries) reflect local time.
A late-night manual scrape and early-morning automated scrape will no
longer both report as the same UTC day.

Updated getTodayUTC/getTodayDateUTC helpers to use toLocaleDateString
with the configured timezone. ISO timestamps for last_scraped, first_seen,
startedAt etc. remain in UTC as they represent precise moments.
This commit is contained in:
Stephen Minakian
2026-02-09 12:06:50 -07:00
parent 40b4dc50f1
commit 4fb7bf833e
4 changed files with 17 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
__tests__/scraper/scraperConfig.test.js
View File

@ -50,15 +50,15 @@ describe('config/scraper', () => {
});
describe('SCRAPER_TIMEZONE', () => {
test('should default to "UTC" when env var not set', () => {
test('should default to "America/Denver" when env var not set', () => {
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('UTC');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
});
test('should use env var override when set', () => {
process.env.SCRAPER_TIMEZONE = 'America/Denver';
process.env.SCRAPER_TIMEZONE = 'America/New_York';
const config = require('../../config/scraper');
expect(config.SCRAPER_TIMEZONE).toBe('America/Denver');
expect(config.SCRAPER_TIMEZONE).toBe('America/New_York');
});
});

2
config/scraper.js
View File

@ -13,7 +13,7 @@ module.exports = {
// Scheduling configuration
SCRAPER_SCHEDULE: process.env.SCRAPER_SCHEDULE || '0 6 * * *',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'UTC',
SCRAPER_TIMEZONE: process.env.SCRAPER_TIMEZONE || 'America/Denver',
SCRAPER_ENABLED: process.env.SCRAPER_ENABLED !== 'false',
// HTTP settings

14
server.js
View File

@ -107,8 +107,8 @@ async function connectToMongoDB() {
// Register scraper signal handlers for graceful shutdown
registerSignalHandlers();
// Helper function to get today's date in UTC
const getTodayDateUTC = () => new Date().toISOString().split('T')[0];
// Helper function to get today's date in configured timezone
const getTodayDate = () => new Date().toLocaleDateString('en-CA', { timeZone: scraperConfig.SCRAPER_TIMEZONE });
// Cache for the most recent date with data
let cachedLatestDate = null;
@ -143,7 +143,7 @@ const getLatestDateWithData = async () => {
}
// Fallback to UTC today if no data found
return getTodayDateUTC();
return getTodayDate();
};
// Helper function to get yesterday relative to the latest date with data
@ -641,10 +641,10 @@ app.get('/analytics', requireAuth, async (req, res) => {
]).toArray();
// 4. Historical Comparison - current vs 30/90/365 day averages
const now = new Date();
const date30 = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date90 = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const date365 = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
const tz = scraperConfig.SCRAPER_TIMEZONE;
const date30 = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const date90 = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const date365 = new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toLocaleDateString('en-CA', { timeZone: tz });
const [currentAvg, avg30, avg90, avg365] = await Promise.all([
db.collection(PRICES_COLLECTION).aggregate([

10
services/scraperService.js
View File

@ -626,11 +626,11 @@ async function updateDailySummary(db, summaryData, logger) {
// ============================================================
/**
* Get today's date in YYYY-MM-DD format (UTC).
* Get today's date in YYYY-MM-DD format using the configured timezone.
* @returns {string} Today's date string
*/
function getTodayUTC() {
return new Date().toISOString().split('T')[0];
function getToday() {
return new Date().toLocaleDateString('en-CA', { timeZone: config.SCRAPER_TIMEZONE });
}
/**
@ -820,7 +820,7 @@ async function runScrape(db, options = {}) {
result.unitsProcessed = units.length;
// Step 4: Database operations
const today = getTodayUTC();
const today = getToday();
// Get yesterday's unit codes for comparison
const yesterdayUnits = await getYesterdayUnitCodes(db, today);
@ -904,7 +904,7 @@ module.exports = {
recordScraperRun,
createScraperIndexes,
// Export helpers for testing
getTodayUTC,
getToday,
getYesterdayUnitCodes,
getYesterday,
parseInteger,