Add scraper management admin page
Some checks failed
CI/CD Pipeline - Apartment Dashboard / Run Linter (push) Successful in 13s
CI/CD Pipeline - Apartment Dashboard / Scan Dependencies (push) Successful in 12s
CI/CD Pipeline - Apartment Dashboard / Build & Push Image (push) Failing after 1m9s
CI/CD Pipeline - Apartment Dashboard / Deploy to Production (push) Has been skipped
Some checks failed
CI/CD Pipeline - Apartment Dashboard / Run Linter (push) Successful in 13s
CI/CD Pipeline - Apartment Dashboard / Scan Dependencies (push) Successful in 12s
CI/CD Pipeline - Apartment Dashboard / Build & Push Image (push) Failing after 1m9s
CI/CD Pipeline - Apartment Dashboard / Deploy to Production (push) Has been skipped
New admin page at /admin/scraper for triggering and monitoring apartment data scrapes. Shows current status, last run summary, schedule info, and run controls with dry-run option.
This commit is contained in:
12
src/App.jsx
12
src/App.jsx
@ -12,6 +12,7 @@ import ActivityStream from './pages/admin/ActivityStream';
|
||||
import AdminOverview from './pages/admin/AdminOverview';
|
||||
import UsageStats from './pages/admin/UsageStats';
|
||||
import AdminSettings from './pages/admin/AdminSettings';
|
||||
import ScraperManagement from './pages/admin/ScraperManagement';
|
||||
|
||||
const ApartmentDashboard = () => {
|
||||
// Router
|
||||
@ -607,6 +608,12 @@ const ApartmentDashboard = () => {
|
||||
>
|
||||
Settings
|
||||
</Link>
|
||||
<Link
|
||||
to="/admin/scraper"
|
||||
className={`block px-4 py-2 text-sm hover:bg-violet-50 ${location.pathname === '/admin/scraper' ? 'bg-violet-100 text-violet-700' : 'text-gray-700'}`}
|
||||
>
|
||||
Scraper
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
@ -1372,6 +1379,11 @@ const ApartmentDashboard = () => {
|
||||
<AdminSettings />
|
||||
</AdminRoute>
|
||||
} />
|
||||
<Route path="/admin/scraper" element={
|
||||
<AdminRoute>
|
||||
<ScraperManagement />
|
||||
</AdminRoute>
|
||||
} />
|
||||
|
||||
{/* Catch-all redirect to overview */}
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
|
||||
298
src/pages/admin/ScraperManagement.jsx
Normal file
298
src/pages/admin/ScraperManagement.jsx
Normal file
@ -0,0 +1,298 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { Play, RefreshCw, Clock, CheckCircle, XCircle, AlertTriangle } from 'lucide-react';
|
||||
|
||||
const API_BASE = 'https://apartments.maverickapplications.com/api';
|
||||
|
||||
const ScraperManagement = () => {
|
||||
const [status, setStatus] = useState(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(null);
|
||||
const [triggering, setTriggering] = useState(false);
|
||||
const [dryRun, setDryRun] = useState(false);
|
||||
const [triggerResult, setTriggerResult] = useState(null);
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/admin/scraper/status`, {
|
||||
credentials: 'include'
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch scraper status');
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setStatus(data);
|
||||
} catch (err) {
|
||||
console.error('Error fetching scraper status:', err);
|
||||
setError(err.message);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchStatus();
|
||||
}, [fetchStatus]);
|
||||
|
||||
// Auto-clear trigger result after 5 seconds
|
||||
useEffect(() => {
|
||||
if (triggerResult) {
|
||||
const timer = setTimeout(() => setTriggerResult(null), 5000);
|
||||
return () => clearTimeout(timer);
|
||||
}
|
||||
}, [triggerResult]);
|
||||
|
||||
const handleRunScraper = async () => {
|
||||
setTriggering(true);
|
||||
setTriggerResult(null);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/admin/scraper/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({ dryRun })
|
||||
});
|
||||
|
||||
if (response.status === 409) {
|
||||
setTriggerResult({ type: 'error', message: 'Scraper is already running' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
throw new Error(errorData.error || 'Failed to trigger scraper');
|
||||
}
|
||||
|
||||
setTriggerResult({ type: 'success', message: `Scraper ${dryRun ? '(dry run) ' : ''}started successfully` });
|
||||
// Re-fetch status after triggering
|
||||
await fetchStatus();
|
||||
} catch (err) {
|
||||
console.error('Error triggering scraper:', err);
|
||||
setTriggerResult({ type: 'error', message: err.message });
|
||||
} finally {
|
||||
setTriggering(false);
|
||||
}
|
||||
};
|
||||
|
||||
const isRunning = status?.currentStatus === 'running';
|
||||
|
||||
const formatDuration = (ms) => {
|
||||
if (!ms) return 'N/A';
|
||||
const seconds = Math.floor(ms / 1000);
|
||||
if (seconds < 60) return `${seconds}s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainingSeconds = seconds % 60;
|
||||
return `${minutes}m ${remainingSeconds}s`;
|
||||
};
|
||||
|
||||
const formatDate = (dateStr) => {
|
||||
if (!dateStr) return 'N/A';
|
||||
return new Date(dateStr).toLocaleString();
|
||||
};
|
||||
|
||||
// Skeleton loader
|
||||
const StatusSkeleton = () => (
|
||||
<div className="space-y-6 animate-pulse">
|
||||
<div className="bg-white rounded-lg shadow-md p-6">
|
||||
<div className="h-6 bg-gray-200 rounded w-48 mb-4"></div>
|
||||
<div className="space-y-4">
|
||||
<div className="h-4 bg-gray-200 rounded w-32"></div>
|
||||
<div className="h-4 bg-gray-200 rounded w-full max-w-md"></div>
|
||||
<div className="h-4 bg-gray-200 rounded w-3/4 max-w-md"></div>
|
||||
<div className="h-10 bg-gray-200 rounded w-32 mt-4"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
if (error && !status) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">Scraper Management</h1>
|
||||
<p className="text-gray-600">Trigger and monitor apartment data scrapes</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-6 text-center">
|
||||
<p className="text-red-700 mb-4">Error: {error}</p>
|
||||
<button
|
||||
onClick={fetchStatus}
|
||||
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700"
|
||||
>
|
||||
Try Again
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">Scraper Management</h1>
|
||||
<p className="text-gray-600">Trigger and monitor apartment data scrapes</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={fetchStatus}
|
||||
disabled={loading}
|
||||
className="flex items-center space-x-2 px-4 py-2 bg-violet-600 text-white rounded-md hover:bg-violet-700 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${loading ? 'animate-spin' : ''}`} />
|
||||
<span>Refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Status Card */}
|
||||
{loading && !status ? (
|
||||
<StatusSkeleton />
|
||||
) : (
|
||||
<div className="bg-white rounded-lg shadow-md p-6">
|
||||
{/* Current Status */}
|
||||
<div className="flex items-center space-x-3 mb-6">
|
||||
<span className="text-lg font-semibold text-gray-900">Status:</span>
|
||||
{isRunning ? (
|
||||
<span className="inline-flex items-center space-x-1.5 px-3 py-1 rounded-full text-sm font-medium bg-amber-100 text-amber-800">
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
<span>Running</span>
|
||||
</span>
|
||||
) : (
|
||||
<span className="inline-flex items-center space-x-1.5 px-3 py-1 rounded-full text-sm font-medium bg-green-100 text-green-800">
|
||||
<CheckCircle className="w-3.5 h-3.5" />
|
||||
<span>Idle</span>
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Last Run Summary */}
|
||||
{status?.lastRun && (
|
||||
<div className="mb-6">
|
||||
<h3 className="text-sm font-semibold text-gray-700 uppercase tracking-wide mb-3">Last Run</h3>
|
||||
<div className="bg-gray-50 rounded-md p-4 space-y-2">
|
||||
<div className="flex items-center space-x-2">
|
||||
<Clock className="w-4 h-4 text-gray-500" />
|
||||
<span className="text-sm text-gray-600">{formatDate(status.lastRun.timestamp)}</span>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
{status.lastRun.status === 'success' ? (
|
||||
<CheckCircle className="w-4 h-4 text-green-500" />
|
||||
) : (
|
||||
<XCircle className="w-4 h-4 text-red-500" />
|
||||
)}
|
||||
<span className={`text-sm font-medium ${status.lastRun.status === 'success' ? 'text-green-700' : 'text-red-700'}`}>
|
||||
{status.lastRun.status === 'success' ? 'Success' : 'Error'}
|
||||
</span>
|
||||
</div>
|
||||
{status.lastRun.duration != null && (
|
||||
<div className="text-sm text-gray-600">
|
||||
<span className="font-medium">Duration:</span> {formatDuration(status.lastRun.duration)}
|
||||
</div>
|
||||
)}
|
||||
{status.lastRun.unitsProcessed != null && (
|
||||
<div className="text-sm text-gray-600">
|
||||
<span className="font-medium">Units processed:</span> {status.lastRun.unitsProcessed}
|
||||
</div>
|
||||
)}
|
||||
{status.lastRun.pricesInserted != null && (
|
||||
<div className="text-sm text-gray-600">
|
||||
<span className="font-medium">Prices inserted:</span> {status.lastRun.pricesInserted}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Schedule Info */}
|
||||
{(status?.schedule || status?.nextScheduledRun) && (
|
||||
<div className="mb-6">
|
||||
<h3 className="text-sm font-semibold text-gray-700 uppercase tracking-wide mb-3">Schedule</h3>
|
||||
<div className="bg-gray-50 rounded-md p-4 space-y-2">
|
||||
{status.schedule && (
|
||||
<div className="text-sm text-gray-600">
|
||||
<span className="font-medium">Cron:</span>{' '}
|
||||
<code className="bg-gray-200 px-1.5 py-0.5 rounded text-xs">{status.schedule}</code>
|
||||
</div>
|
||||
)}
|
||||
{status.nextScheduledRun && (
|
||||
<div className="flex items-center space-x-2">
|
||||
<Clock className="w-4 h-4 text-gray-500" />
|
||||
<span className="text-sm text-gray-600">
|
||||
<span className="font-medium">Next run:</span> {formatDate(status.nextScheduledRun)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Divider */}
|
||||
<hr className="my-6 border-gray-200" />
|
||||
|
||||
{/* Run Controls */}
|
||||
<div className="space-y-4">
|
||||
<h3 className="text-sm font-semibold text-gray-700 uppercase tracking-wide">Run Controls</h3>
|
||||
|
||||
<div className="flex items-center space-x-4">
|
||||
<button
|
||||
onClick={handleRunScraper}
|
||||
disabled={triggering || isRunning}
|
||||
className="flex items-center space-x-2 px-4 py-2 bg-violet-600 text-white rounded-md hover:bg-violet-700 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{triggering ? (
|
||||
<>
|
||||
<RefreshCw className="w-4 h-4 animate-spin" />
|
||||
<span>Starting...</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Play className="w-4 h-4" />
|
||||
<span>Run Scraper</span>
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
|
||||
<label className="flex items-center space-x-2 cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={dryRun}
|
||||
onChange={(e) => setDryRun(e.target.checked)}
|
||||
className="w-4 h-4 text-violet-600 border-gray-300 rounded focus:ring-violet-500"
|
||||
/>
|
||||
<span className="text-sm text-gray-700">Dry run</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Trigger Result Message */}
|
||||
{triggerResult && (
|
||||
<div className={`flex items-center space-x-2 p-3 rounded-md border ${
|
||||
triggerResult.type === 'success'
|
||||
? 'bg-green-50 border-green-200'
|
||||
: 'bg-red-50 border-red-200'
|
||||
}`}>
|
||||
{triggerResult.type === 'success' ? (
|
||||
<CheckCircle className="w-4 h-4 text-green-600 flex-shrink-0" />
|
||||
) : (
|
||||
<AlertTriangle className="w-4 h-4 text-red-600 flex-shrink-0" />
|
||||
)}
|
||||
<span className={`text-sm ${
|
||||
triggerResult.type === 'success' ? 'text-green-700' : 'text-red-700'
|
||||
}`}>
|
||||
{triggerResult.message}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ScraperManagement;
|
||||
Reference in New Issue
Block a user