feat: sanitize error messages before recording to scraper_runs
All checks were successful
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 44s
All checks were successful
CI/CD Pipeline - Apartment API / Send Webhook Notification (pull_request) Successful in 2s
CI/CD Pipeline - Apartment API / Build & Push Image (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Deploy to Production (pull_request) Has been skipped
CI/CD Pipeline - Apartment API / Scan Dependencies (pull_request) Successful in 13s
CI/CD Pipeline - Apartment API / Lint & Test (pull_request) Successful in 44s
Add sanitizeError() and sanitizeMessage() functions that strip sensitive information from error messages before they are stored in scraper_runs history. This prevents accidental exposure of infrastructure details in the database. Sanitization covers: - Unix and Windows file paths (e.g., /home/deploy/app/..., C:\Users\...) - MongoDB connection strings (mongodb:// and mongodb+srv://) - Credential patterns (API_KEY=, password=, secret=, token=) - Stack trace file path references The error type/name (e.g., MongoServerError, TypeError) and general debugging description are preserved to maintain diagnostic usefulness. Applied in runScrape() catch block before calling recordScraperRun(), ensuring only sanitized messages reach the database. Added 18 new tests covering all sanitization categories: file paths, connection strings, credentials, error type preservation, description preservation, stack trace removal, and integration with recordScraperRun.
This commit is contained in:
@ -651,6 +651,54 @@ async function getYesterdayUnitCodes(db, today) {
|
||||
return new Set(yesterdayRecords.map(r => r.unit_code));
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Error Sanitization
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Sanitize an error object to remove sensitive information before storage.
|
||||
* Removes file paths, connection strings, and credential patterns while
|
||||
* preserving the error type and a useful general description for debugging.
|
||||
*
|
||||
* @param {Error} error - Error object to sanitize
|
||||
* @returns {Object} Sanitized error with name, message, and optionally stack
|
||||
*/
|
||||
function sanitizeError(error) {
|
||||
const sanitized = {
|
||||
name: error.name || 'Error',
|
||||
message: sanitizeMessage(error.message || ''),
|
||||
};
|
||||
|
||||
if (error.stack) {
|
||||
sanitized.stack = sanitizeMessage(error.stack);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a string message by removing sensitive patterns.
|
||||
* @param {string} message - Raw error message
|
||||
* @returns {string} Sanitized message
|
||||
*/
|
||||
function sanitizeMessage(message) {
|
||||
let result = message;
|
||||
|
||||
// Redact MongoDB connection strings (mongodb:// and mongodb+srv://)
|
||||
result = result.replace(/mongodb(\+srv)?:\/\/[^\s,;)}\]'"]+/gi, '[REDACTED_CONNECTION_STRING]');
|
||||
|
||||
// Redact credential/secret patterns: KEY=value, password=value, token=value, etc.
|
||||
result = result.replace(/\b(api[_-]?key|secret[_-]?key|secret[_-]?token|token|password|passwd|authorization|credential)\s*=\s*\S+/gi, '$1=[REDACTED]');
|
||||
|
||||
// Remove Unix absolute paths (/home/..., /var/..., /tmp/..., /usr/..., /etc/..., /opt/...)
|
||||
result = result.replace(/\/(?:home|var|tmp|usr|etc|opt)\/[^\s:,;)}\]'"]+/g, '[PATH]');
|
||||
|
||||
// Remove Windows-style absolute paths (C:\..., D:\...)
|
||||
result = result.replace(/[A-Z]:\\[^\s:,;)}\]'"]+/gi, '[PATH]');
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Scraper Run History
|
||||
// ============================================================
|
||||
@ -787,12 +835,13 @@ async function runScrape(db, options = {}) {
|
||||
result.status = 'success';
|
||||
|
||||
} catch (error) {
|
||||
const cleanError = sanitizeError(error);
|
||||
logger.error('Scrape failed', {
|
||||
errorType: error.name,
|
||||
errorMessage: error.message
|
||||
errorType: cleanError.name,
|
||||
errorMessage: cleanError.message
|
||||
});
|
||||
result.status = 'failed';
|
||||
result.errors.push(error.message);
|
||||
result.errors.push(cleanError.message);
|
||||
|
||||
} finally {
|
||||
result.completedAt = new Date().toISOString();
|
||||
@ -834,5 +883,6 @@ module.exports = {
|
||||
parseFloatValue,
|
||||
trimString,
|
||||
isRetryableError,
|
||||
sleep
|
||||
sleep,
|
||||
sanitizeError
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user