__tests__/scraper/scraperLogger.test.js

@@ -0,0 +1,471 @@
+/**
+ * Tests for scraperLogger.js
+ * Following TDD - these tests are written BEFORE implementation
+ */
+
+const { createLogger, LEVELS } = require('../../services/scraperLogger');
+
+describe('scraperLogger', () => {
+  let consoleSpy;
+  let logOutput;
+
+  beforeEach(() => {
+    logOutput = [];
+    consoleSpy = jest.spyOn(console, 'log').mockImplementation((output) => {
+      logOutput.push(output);
+    });
+  });
+
+  afterEach(() => {
+    consoleSpy.mockRestore();
+  });
+
+  describe('createLogger', () => {
+    test('should return an object with info, warn, error methods', () => {
+      const logger = createLogger('test-job-id');
+
+      expect(typeof logger).toBe('object');
+      expect(typeof logger.info).toBe('function');
+      expect(typeof logger.warn).toBe('function');
+      expect(typeof logger.error).toBe('function');
+    });
+
+    test('should include jobId in all log entries', () => {
+      const jobId = 'unique-job-123';
+      const logger = createLogger(jobId);
+
+      logger.info('Test message');
+
+      expect(logOutput.length).toBe(1);
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.jobId).toBe(jobId);
+    });
+  });
+
+  describe('log entry format', () => {
+    test('should include timestamp in ISO format', () => {
+      const logger = createLogger('test-job');
+      const beforeTime = new Date().toISOString();
+
+      logger.info('Test message');
+
+      const afterTime = new Date().toISOString();
+      const parsed = JSON.parse(logOutput[0]);
+
+      expect(parsed.timestamp).toBeDefined();
+      // Verify timestamp is in ISO format
+      expect(parsed.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
+      // Verify timestamp is within test window
+      expect(parsed.timestamp >= beforeTime).toBe(true);
+      expect(parsed.timestamp <= afterTime).toBe(true);
+    });
+
+    test('should include log level', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+      logger.warn('Warn message');
+      logger.error('Error message');
+
+      expect(logOutput.length).toBe(3);
+
+      const infoEntry = JSON.parse(logOutput[0]);
+      const warnEntry = JSON.parse(logOutput[1]);
+      const errorEntry = JSON.parse(logOutput[2]);
+
+      expect(infoEntry.level).toBe('info');
+      expect(warnEntry.level).toBe('warn');
+      expect(errorEntry.level).toBe('error');
+    });
+
+    test('should include message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message content');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('Test message content');
+    });
+
+    test('should include context when provided', () => {
+      const logger = createLogger('test-job');
+      const context = { key: 'value', count: 42 };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual(context);
+    });
+
+    test('should output valid JSON', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', { data: 'test' });
+
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('log levels', () => {
+    test('info() should output with level "info"', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Info message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('info');
+    });
+
+    test('warn() should output with level "warn"', () => {
+      const logger = createLogger('test-job');
+
+      logger.warn('Warning message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('warn');
+    });
+
+    test('error() should output with level "error"', () => {
+      const logger = createLogger('test-job');
+
+      logger.error('Error message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.level).toBe('error');
+    });
+  });
+
+  describe('LEVELS constant', () => {
+    test('should export LEVELS constant with info, warn, error', () => {
+      expect(LEVELS).toBeDefined();
+      expect(LEVELS.info).toBe('info');
+      expect(LEVELS.warn).toBe('warn');
+      expect(LEVELS.error).toBe('error');
+    });
+  });
+
+  describe('message truncation', () => {
+    test('should truncate messages longer than 1000 characters', () => {
+      const logger = createLogger('test-job');
+      const longMessage = 'x'.repeat(1500);
+
+      logger.info(longMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message.length).toBeLessThanOrEqual(1020); // 1000 + "... [truncated]"
+      expect(parsed.message).toContain('... [truncated]');
+    });
+
+    test('should not truncate messages of 1000 characters or less', () => {
+      const logger = createLogger('test-job');
+      const shortMessage = 'x'.repeat(1000);
+
+      logger.info(shortMessage);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe(shortMessage);
+      expect(parsed.message).not.toContain('... [truncated]');
+    });
+
+    test('should handle empty message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.message).toBe('');
+    });
+
+    test('should handle null or undefined message', () => {
+      const logger = createLogger('test-job');
+
+      logger.info(null);
+      logger.info(undefined);
+
+      expect(logOutput.length).toBe(2);
+      // Should not throw and should handle gracefully
+    });
+  });
+
+  describe('MongoDB connection string redaction', () => {
+    test('should redact username and password from mongodb:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://admin:secretpassword123@localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://[user]:[REDACTED]@localhost:27017/mydb');
+      expect(parsed.context.uri).not.toContain('admin');
+      expect(parsed.context.uri).not.toContain('secretpassword123');
+    });
+
+    test('should redact username and password from mongodb+srv:// URIs', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb+srv://myuser:mypass@cluster0.example.mongodb.net/testdb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb+srv://[user]:[REDACTED]@cluster0.example.mongodb.net/testdb');
+      expect(parsed.context.uri).not.toContain('myuser');
+      expect(parsed.context.uri).not.toContain('mypass');
+    });
+
+    test('should handle connection string without credentials', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        uri: 'mongodb://localhost:27017/mydb'
+      };
+
+      logger.info('Database connection', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.uri).toBe('mongodb://localhost:27017/mydb');
+    });
+  });
+
+  describe('sensitive key redaction', () => {
+    test('should redact keys containing "password"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        password: 'secret123',
+        userPassword: 'anotherSecret',
+        password_hash: 'hashed'
+      };
+
+      logger.info('User data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.password).toBe('[REDACTED]');
+      expect(parsed.context.userPassword).toBe('[REDACTED]');
+      expect(parsed.context.password_hash).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "secret"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        secret: 'mysecret',
+        clientSecret: 'secret123',
+        SECRET_KEY: 'key'
+      };
+
+      logger.info('Config data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.secret).toBe('[REDACTED]');
+      expect(parsed.context.clientSecret).toBe('[REDACTED]');
+      // Case-insensitive check
+    });
+
+    test('should redact keys containing "token"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        token: 'abc123',
+        accessToken: 'token456',
+        refresh_token: 'refresh789'
+      };
+
+      logger.info('Auth data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.token).toBe('[REDACTED]');
+      expect(parsed.context.accessToken).toBe('[REDACTED]');
+      expect(parsed.context.refresh_token).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "apikey"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        apikey: 'key123',
+        apiKey: 'key456',
+        api_key: 'key789' // Note: underscores may not match 'apikey'
+      };
+
+      logger.info('API data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.apikey).toBe('[REDACTED]');
+      expect(parsed.context.apiKey).toBe('[REDACTED]');
+    });
+
+    test('should redact keys containing "authorization"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        authorization: 'Bearer token123',
+        Authorization: 'Basic base64string'
+      };
+
+      logger.info('Header data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.authorization).toBe('[REDACTED]');
+    });
+
+    test('should not redact non-sensitive keys', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        username: 'john',
+        email: 'john@example.com',
+        count: 42
+      };
+
+      logger.info('User info', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.username).toBe('john');
+      expect(parsed.context.email).toBe('john@example.com');
+      expect(parsed.context.count).toBe(42);
+    });
+  });
+
+  describe('circular reference handling', () => {
+    test('should handle circular references in context', () => {
+      const logger = createLogger('test-job');
+      const context = { name: 'test' };
+      context.self = context; // Create circular reference
+
+      // Should not throw
+      expect(() => {
+        logger.info('Circular test', context);
+      }).not.toThrow();
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.name).toBe('test');
+      expect(parsed.context.self).toBe('[Circular]');
+    });
+
+    test('should handle deeply nested circular references', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        level1: {
+          level2: {
+            level3: {}
+          }
+        }
+      };
+      context.level1.level2.level3.back = context.level1;
+
+      expect(() => {
+        logger.info('Deep circular test', context);
+      }).not.toThrow();
+
+      // Should be valid JSON output
+      expect(() => JSON.parse(logOutput[0])).not.toThrow();
+    });
+  });
+
+  describe('Buffer handling', () => {
+    test('should represent Buffer objects as "[Buffer: N bytes]"', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.from('Hello World')
+      };
+
+      logger.info('Buffer data', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 11 bytes]');
+    });
+
+    test('should handle empty Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(0)
+      };
+
+      logger.info('Empty buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 0 bytes]');
+    });
+
+    test('should handle large Buffer', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        data: Buffer.alloc(1024 * 1024) // 1MB buffer
+      };
+
+      logger.info('Large buffer', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.data).toBe('[Buffer: 1048576 bytes]');
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty context object', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', {});
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle context not provided', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message');
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle null context', () => {
+      const logger = createLogger('test-job');
+
+      logger.info('Test message', null);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context).toEqual({});
+    });
+
+    test('should handle undefined values in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        defined: 'value',
+        undefinedValue: undefined
+      };
+
+      logger.info('Test message', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.defined).toBe('value');
+      // undefined is typically converted to null in JSON
+      expect(parsed.context.undefinedValue).toBeNull();
+    });
+
+    test('should handle nested objects in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        outer: {
+          inner: {
+            value: 'deep'
+          }
+        }
+      };
+
+      logger.info('Nested test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.outer.inner.value).toBe('deep');
+    });
+
+    test('should handle arrays in context', () => {
+      const logger = createLogger('test-job');
+      const context = {
+        items: [1, 2, 3, 'four']
+      };
+
+      logger.info('Array test', context);
+
+      const parsed = JSON.parse(logOutput[0]);
+      expect(parsed.context.items).toEqual([1, 2, 3, 'four']);
+    });
+  });
+});

package-lock.json

@@ -9,12 +9,16 @@
             "version": "1.0.0",
             "license": "MIT",
             "dependencies": {
+                "axios": "^1.13.4",
+                "cheerio": "^1.2.0",
                 "cookie-parser": "^1.4.7",
                 "cors": "^2.8.5",
+                "cron-parser": "^5.5.0",
                 "express": "^4.18.2",
                 "express-rate-limit": "^7.1.5",
                 "jsonwebtoken": "^9.0.3",
                 "mongodb": "^6.3.0",
+                "node-cron": "^4.2.1",
                 "passport": "^0.7.0",
                 "passport-google-oauth20": "^2.0.0",
                 "uuid": "^13.0.0"
@@ -1689,9 +1693,19 @@
             "version": "0.4.0",
             "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
             "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-            "dev": true,
             "license": "MIT"
         },
+        "node_modules/axios": {
+            "version": "1.13.4",
+            "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.4.tgz",
+            "integrity": "sha512-1wVkUaAO6WyaYtCkcYCOx12ZgpGf9Zif+qXa4n+oYzK558YryKqiL6UWwd5DqiH3VRW0GYhTZQ/vlgJrCoNQlg==",
+            "license": "MIT",
+            "dependencies": {
+                "follow-redirects": "^1.15.6",
+                "form-data": "^4.0.4",
+                "proxy-from-env": "^1.1.0"
+            }
+        },
         "node_modules/b4a": {
             "version": "1.7.3",
             "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz",
@@ -1913,6 +1927,12 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/boolbase": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+            "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
+            "license": "ISC"
+        },
         "node_modules/brace-expansion": {
             "version": "1.1.12",
             "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
@@ -2142,6 +2162,48 @@
                 "node": ">=10"
             }
         },
+        "node_modules/cheerio": {
+            "version": "1.2.0",
+            "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
+            "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
+            "license": "MIT",
+            "dependencies": {
+                "cheerio-select": "^2.1.0",
+                "dom-serializer": "^2.0.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.2.2",
+                "encoding-sniffer": "^0.2.1",
+                "htmlparser2": "^10.1.0",
+                "parse5": "^7.3.0",
+                "parse5-htmlparser2-tree-adapter": "^7.1.0",
+                "parse5-parser-stream": "^7.1.2",
+                "undici": "^7.19.0",
+                "whatwg-mimetype": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=20.18.1"
+            },
+            "funding": {
+                "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+            }
+        },
+        "node_modules/cheerio-select": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+            "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "boolbase": "^1.0.0",
+                "css-select": "^5.1.0",
+                "css-what": "^6.1.0",
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.0.1"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/fb55"
+            }
+        },
         "node_modules/chokidar": {
             "version": "3.6.0",
             "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
@@ -2310,7 +2372,6 @@
             "version": "1.0.8",
             "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
             "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-            "dev": true,
             "license": "MIT",
             "dependencies": {
                 "delayed-stream": "~1.0.0"
@@ -2428,6 +2489,18 @@
                 "node": ">= 0.10"
             }
         },
+        "node_modules/cron-parser": {
+            "version": "5.5.0",
+            "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.5.0.tgz",
+            "integrity": "sha512-oML4lKUXxizYswqmxuOCpgFS8BNUJpIu6k/2HVHyaL8Ynnf3wdf9tkns0yRdJLSIjkJ+b0DXHMZEHGpMwjnPww==",
+            "license": "MIT",
+            "dependencies": {
+                "luxon": "^3.7.1"
+            },
+            "engines": {
+                "node": ">=18"
+            }
+        },
         "node_modules/cross-spawn": {
             "version": "7.0.6",
             "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -2443,6 +2516,34 @@
                 "node": ">= 8"
             }
         },
+        "node_modules/css-select": {
+            "version": "5.2.2",
+            "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
+            "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "boolbase": "^1.0.0",
+                "css-what": "^6.1.0",
+                "domhandler": "^5.0.2",
+                "domutils": "^3.0.1",
+                "nth-check": "^2.0.1"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/fb55"
+            }
+        },
+        "node_modules/css-what": {
+            "version": "6.2.2",
+            "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
+            "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
+            "license": "BSD-2-Clause",
+            "engines": {
+                "node": ">= 6"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/fb55"
+            }
+        },
         "node_modules/debug": {
             "version": "2.6.9",
             "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -2481,7 +2582,6 @@
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
             "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-            "dev": true,
             "license": "MIT",
             "engines": {
                 "node": ">=0.4.0"
@@ -2527,6 +2627,61 @@
                 "wrappy": "1"
             }
         },
+        "node_modules/dom-serializer": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+            "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+            "license": "MIT",
+            "dependencies": {
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.2",
+                "entities": "^4.2.0"
+            },
+            "funding": {
+                "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+            }
+        },
+        "node_modules/domelementtype": {
+            "version": "2.3.0",
+            "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+            "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+            "funding": [
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/fb55"
+                }
+            ],
+            "license": "BSD-2-Clause"
+        },
+        "node_modules/domhandler": {
+            "version": "5.0.3",
+            "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+            "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "domelementtype": "^2.3.0"
+            },
+            "engines": {
+                "node": ">= 4"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/domhandler?sponsor=1"
+            }
+        },
+        "node_modules/domutils": {
+            "version": "3.2.2",
+            "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+            "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "dom-serializer": "^2.0.0",
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.3"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/domutils?sponsor=1"
+            }
+        },
         "node_modules/dunder-proto": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -2599,6 +2754,43 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/encoding-sniffer": {
+            "version": "0.2.1",
+            "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+            "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+            "license": "MIT",
+            "dependencies": {
+                "iconv-lite": "^0.6.3",
+                "whatwg-encoding": "^3.1.1"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+            }
+        },
+        "node_modules/encoding-sniffer/node_modules/iconv-lite": {
+            "version": "0.6.3",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+            "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+            "license": "MIT",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
+        "node_modules/entities": {
+            "version": "4.5.0",
+            "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+            "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+            "license": "BSD-2-Clause",
+            "engines": {
+                "node": ">=0.12"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/entities?sponsor=1"
+            }
+        },
         "node_modules/error-ex": {
             "version": "1.3.4",
             "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz",
@@ -2643,7 +2835,6 @@
             "version": "2.1.0",
             "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
             "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
-            "dev": true,
             "license": "MIT",
             "dependencies": {
                 "es-errors": "^1.3.0",
@@ -2958,7 +3149,6 @@
             "version": "1.15.11",
             "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
             "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
-            "dev": true,
             "funding": [
                 {
                     "type": "individual",
@@ -2996,7 +3186,6 @@
             "version": "4.0.5",
             "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
             "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
-            "dev": true,
             "license": "MIT",
             "dependencies": {
                 "asynckit": "^0.4.0",
@@ -3261,7 +3450,6 @@
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
             "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
-            "dev": true,
             "license": "MIT",
             "dependencies": {
                 "has-symbols": "^1.0.3"
@@ -3292,6 +3480,37 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/htmlparser2": {
+            "version": "10.1.0",
+            "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
+            "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
+            "funding": [
+                "https://github.com/fb55/htmlparser2?sponsor=1",
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/fb55"
+                }
+            ],
+            "license": "MIT",
+            "dependencies": {
+                "domelementtype": "^2.3.0",
+                "domhandler": "^5.0.3",
+                "domutils": "^3.2.2",
+                "entities": "^7.0.1"
+            }
+        },
+        "node_modules/htmlparser2/node_modules/entities": {
+            "version": "7.0.1",
+            "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
+            "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
+            "license": "BSD-2-Clause",
+            "engines": {
+                "node": ">=0.12"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/entities?sponsor=1"
+            }
+        },
         "node_modules/http-errors": {
             "version": "2.0.0",
             "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -4449,6 +4668,15 @@
                 "yallist": "^3.0.2"
             }
         },
+        "node_modules/luxon": {
+            "version": "3.7.2",
+            "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz",
+            "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==",
+            "license": "MIT",
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/make-dir": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz",
@@ -4894,6 +5122,15 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/node-cron": {
+            "version": "4.2.1",
+            "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz",
+            "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==",
+            "license": "ISC",
+            "engines": {
+                "node": ">=6.0.0"
+            }
+        },
         "node_modules/node-int64": {
             "version": "0.4.0",
             "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@@ -4985,6 +5222,18 @@
                 "node": ">=8"
             }
         },
+        "node_modules/nth-check": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+            "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "boolbase": "^1.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/nth-check?sponsor=1"
+            }
+        },
         "node_modules/oauth": {
             "version": "0.10.2",
             "resolved": "https://registry.npmjs.org/oauth/-/oauth-0.10.2.tgz",
@@ -5131,6 +5380,55 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/parse5": {
+            "version": "7.3.0",
+            "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+            "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+            "license": "MIT",
+            "dependencies": {
+                "entities": "^6.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/inikulin/parse5?sponsor=1"
+            }
+        },
+        "node_modules/parse5-htmlparser2-tree-adapter": {
+            "version": "7.1.0",
+            "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+            "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+            "license": "MIT",
+            "dependencies": {
+                "domhandler": "^5.0.3",
+                "parse5": "^7.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/inikulin/parse5?sponsor=1"
+            }
+        },
+        "node_modules/parse5-parser-stream": {
+            "version": "7.1.2",
+            "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+            "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+            "license": "MIT",
+            "dependencies": {
+                "parse5": "^7.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/inikulin/parse5?sponsor=1"
+            }
+        },
+        "node_modules/parse5/node_modules/entities": {
+            "version": "6.0.1",
+            "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+            "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+            "license": "BSD-2-Clause",
+            "engines": {
+                "node": ">=0.12"
+            },
+            "funding": {
+                "url": "https://github.com/fb55/entities?sponsor=1"
+            }
+        },
         "node_modules/parseurl": {
             "version": "1.3.3",
             "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -5354,6 +5652,12 @@
                 "node": ">= 0.10"
             }
         },
+        "node_modules/proxy-from-env": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+            "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
+            "license": "MIT"
+        },
         "node_modules/pstree.remy": {
             "version": "1.1.8",
             "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
@@ -6261,6 +6565,15 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/undici": {
+            "version": "7.19.2",
+            "resolved": "https://registry.npmjs.org/undici/-/undici-7.19.2.tgz",
+            "integrity": "sha512-4VQSpGEGsWzk0VYxyB/wVX/Q7qf9t5znLRgs0dzszr9w9Fej/8RVNQ+S20vdXSAyra/bJ7ZQfGv6ZMj7UEbzSg==",
+            "license": "MIT",
+            "engines": {
+                "node": ">=20.18.1"
+            }
+        },
         "node_modules/undici-types": {
             "version": "7.16.0",
             "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
@@ -6408,6 +6721,40 @@
                 "node": ">=12"
             }
         },
+        "node_modules/whatwg-encoding": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+            "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+            "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
+            "license": "MIT",
+            "dependencies": {
+                "iconv-lite": "0.6.3"
+            },
+            "engines": {
+                "node": ">=18"
+            }
+        },
+        "node_modules/whatwg-encoding/node_modules/iconv-lite": {
+            "version": "0.6.3",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+            "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+            "license": "MIT",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
+        "node_modules/whatwg-mimetype": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+            "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            }
+        },
         "node_modules/whatwg-url": {
             "version": "14.2.0",
             "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",

package.json

@@ -19,12 +19,16 @@
     "author": "Stephen",
     "license": "MIT",
     "dependencies": {
+        "axios": "^1.13.4",
+        "cheerio": "^1.2.0",
         "cookie-parser": "^1.4.7",
         "cors": "^2.8.5",
+        "cron-parser": "^5.5.0",
         "express": "^4.18.2",
         "express-rate-limit": "^7.1.5",
         "jsonwebtoken": "^9.0.3",
         "mongodb": "^6.3.0",
+        "node-cron": "^4.2.1",
         "passport": "^0.7.0",
         "passport-google-oauth20": "^2.0.0",
         "uuid": "^13.0.0"

services/scraperLogger.js

@@ -0,0 +1,164 @@
+/**
+ * Scraper-specific structured JSON logger
+ * Produces one JSON object per line to stdout
+ */
+
+const LEVELS = {
+  info: 'info',
+  warn: 'warn',
+  error: 'error'
+};
+
+/**
+ * Create a logger instance scoped to a job ID
+ * @param {string} jobId - Job identifier for correlation
+ * @returns {Object} Logger object with info, warn, error methods
+ */
+function createLogger(jobId) {
+  /**
+   * Internal log function
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} context - Additional context data
+   */
+  const log = (level, message, context = {}) => {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      message: truncateMessage(message),
+      jobId,
+      context: sanitizeContext(context)
+    };
+
+    // Output as single-line JSON
+    console.log(JSON.stringify(entry));
+  };
+
+  return {
+    info: (message, context) => log(LEVELS.info, message, context),
+    warn: (message, context) => log(LEVELS.warn, message, context),
+    error: (message, context) => log(LEVELS.error, message, context)
+  };
+}
+
+/**
+ * Truncate message to prevent log bloat
+ * @param {string} message - Message to truncate
+ * @param {number} maxLength - Maximum length (default 1000)
+ * @returns {string} Truncated message
+ */
+function truncateMessage(message, maxLength = 1000) {
+  if (message === null || message === undefined) {
+    return '';
+  }
+  const str = String(message);
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.substring(0, maxLength) + '... [truncated]';
+}
+
+/**
+ * Recursively process an object to handle Buffers, circular references, and sensitive data
+ * @param {*} obj - Object to process
+ * @param {WeakSet} seen - Set of seen objects for circular reference detection
+ * @returns {*} Processed value
+ */
+function processValue(obj, seen = new WeakSet()) {
+  // Handle null/undefined
+  if (obj === null) {
+    return null;
+  }
+  if (obj === undefined) {
+    return null;
+  }
+
+  // Handle Buffer BEFORE checking for object (Buffer is an object)
+  if (Buffer.isBuffer(obj)) {
+    return `[Buffer: ${obj.length} bytes]`;
+  }
+
+  // Handle strings - check for MongoDB connection strings
+  if (typeof obj === 'string') {
+    if (/mongodb(\+srv)?:\/\//.test(obj)) {
+      return redactConnectionString(obj);
+    }
+    return obj;
+  }
+
+  // Handle primitives
+  if (typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Handle circular references
+  if (seen.has(obj)) {
+    return '[Circular]';
+  }
+  seen.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => processValue(item, seen));
+  }
+
+  // Handle plain objects
+  const result = {};
+  const sensitiveKeys = ['password', 'secret', 'token', 'apikey', 'authorization'];
+
+  for (const key of Object.keys(obj)) {
+    // Check for sensitive keys
+    if (sensitiveKeys.some(k => key.toLowerCase().includes(k))) {
+      result[key] = '[REDACTED]';
+    } else {
+      result[key] = processValue(obj[key], seen);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sanitize context object for safe logging
+ * - Remove circular references
+ * - Redact sensitive data
+ * - Handle special types (Buffer, undefined)
+ * @param {Object} context - Context object
+ * @returns {Object} Sanitized context
+ */
+function sanitizeContext(context) {
+  if (!context || typeof context !== 'object') {
+    return {};
+  }
+
+  try {
+    return processValue(context);
+  } catch (error) {
+    // If sanitization fails, return empty context
+    return { sanitizationError: 'Failed to sanitize context' };
+  }
+}
+
+/**
+ * Redact credentials from MongoDB connection string
+ * @param {string} uri - Connection string
+ * @returns {string} Redacted string
+ */
+function redactConnectionString(uri) {
+  try {
+    // Match mongodb://user:pass@host or mongodb+srv://user:pass@host
+    return uri.replace(
+      /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/,
+      'mongodb$1://[user]:[REDACTED]@'
+    );
+  } catch {
+    return '[REDACTED CONNECTION STRING]';
+  }
+}
+
+module.exports = {
+  createLogger,
+  LEVELS,
+  // Export internal functions for testing
+  redactConnectionString
+};