Add scraper run history table to Scraper Management page
All checks were successful
CI/CD Pipeline - Apartment Dashboard / Build & Push Image (push) Successful in 58s
CI/CD Pipeline - Apartment Dashboard / Deploy to Production (push) Successful in 14s
CI/CD Pipeline - Apartment Dashboard / Run Linter (push) Successful in 12s
CI/CD Pipeline - Apartment Dashboard / Scan Dependencies (push) Successful in 13s
All checks were successful
CI/CD Pipeline - Apartment Dashboard / Build & Push Image (push) Successful in 58s
CI/CD Pipeline - Apartment Dashboard / Deploy to Production (push) Successful in 14s
CI/CD Pipeline - Apartment Dashboard / Run Linter (push) Successful in 12s
CI/CD Pipeline - Apartment Dashboard / Scan Dependencies (push) Successful in 13s
Fetches last 30 scrape events from /api/admin/scraper/history and displays them in a table with time, status, trigger type, duration, units processed, prices inserted, and dry run columns. History refreshes on page load, manual refresh, and after triggering a scrape.
This commit is contained in:
@ -10,6 +10,8 @@ const ScraperManagement = () => {
|
|||||||
const [triggering, setTriggering] = useState(false);
|
const [triggering, setTriggering] = useState(false);
|
||||||
const [dryRun, setDryRun] = useState(false);
|
const [dryRun, setDryRun] = useState(false);
|
||||||
const [triggerResult, setTriggerResult] = useState(null);
|
const [triggerResult, setTriggerResult] = useState(null);
|
||||||
|
const [history, setHistory] = useState([]);
|
||||||
|
const [historyLoading, setHistoryLoading] = useState(true);
|
||||||
|
|
||||||
const fetchStatus = useCallback(async () => {
|
const fetchStatus = useCallback(async () => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
@ -34,9 +36,31 @@ const ScraperManagement = () => {
|
|||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const fetchHistory = useCallback(async () => {
|
||||||
|
setHistoryLoading(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`${API_BASE}/admin/scraper/history`, {
|
||||||
|
credentials: 'include'
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error('Failed to fetch scraper history');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await response.json();
|
||||||
|
setHistory(data.data || []);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error fetching scraper history:', err);
|
||||||
|
} finally {
|
||||||
|
setHistoryLoading(false);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
fetchStatus();
|
fetchStatus();
|
||||||
}, [fetchStatus]);
|
fetchHistory();
|
||||||
|
}, [fetchStatus, fetchHistory]);
|
||||||
|
|
||||||
// Auto-clear trigger result after 5 seconds
|
// Auto-clear trigger result after 5 seconds
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@ -69,8 +93,8 @@ const ScraperManagement = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setTriggerResult({ type: 'success', message: `Scraper ${dryRun ? '(dry run) ' : ''}started successfully` });
|
setTriggerResult({ type: 'success', message: `Scraper ${dryRun ? '(dry run) ' : ''}started successfully` });
|
||||||
// Re-fetch status after triggering
|
// Re-fetch status and history after triggering
|
||||||
await fetchStatus();
|
await Promise.all([fetchStatus(), fetchHistory()]);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error triggering scraper:', err);
|
console.error('Error triggering scraper:', err);
|
||||||
setTriggerResult({ type: 'error', message: err.message });
|
setTriggerResult({ type: 'error', message: err.message });
|
||||||
@ -122,7 +146,7 @@ const ScraperManagement = () => {
|
|||||||
<div className="bg-red-50 border border-red-200 rounded-lg p-6 text-center">
|
<div className="bg-red-50 border border-red-200 rounded-lg p-6 text-center">
|
||||||
<p className="text-red-700 mb-4">Error: {error}</p>
|
<p className="text-red-700 mb-4">Error: {error}</p>
|
||||||
<button
|
<button
|
||||||
onClick={fetchStatus}
|
onClick={() => { fetchStatus(); fetchHistory(); }}
|
||||||
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700"
|
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700"
|
||||||
>
|
>
|
||||||
Try Again
|
Try Again
|
||||||
@ -141,7 +165,7 @@ const ScraperManagement = () => {
|
|||||||
<p className="text-gray-600">Trigger and monitor apartment data scrapes</p>
|
<p className="text-gray-600">Trigger and monitor apartment data scrapes</p>
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
onClick={fetchStatus}
|
onClick={() => { fetchStatus(); fetchHistory(); }}
|
||||||
disabled={loading}
|
disabled={loading}
|
||||||
className="flex items-center space-x-2 px-4 py-2 bg-violet-600 text-white rounded-md hover:bg-violet-700 disabled:opacity-50"
|
className="flex items-center space-x-2 px-4 py-2 bg-violet-600 text-white rounded-md hover:bg-violet-700 disabled:opacity-50"
|
||||||
>
|
>
|
||||||
@ -291,6 +315,100 @@ const ScraperManagement = () => {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Run History */}
|
||||||
|
<div className="bg-white rounded-lg shadow-md overflow-hidden">
|
||||||
|
<div className="px-6 py-4 border-b border-gray-200">
|
||||||
|
<h2 className="text-lg font-semibold text-gray-900">Run History</h2>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{historyLoading && history.length === 0 ? (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full min-w-max">
|
||||||
|
<thead className="bg-gray-50 border-b">
|
||||||
|
<tr>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700">Time</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Status</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Trigger</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Duration</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Units</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Prices</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Dry Run</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{[...Array(5)].map((_, index) => (
|
||||||
|
<tr key={index} className={`border-b ${index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}`}>
|
||||||
|
<td className="py-3 px-4"><div className="h-4 bg-gray-200 rounded w-36 animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4 text-center"><div className="h-5 bg-gray-200 rounded-full w-16 mx-auto animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4 text-center"><div className="h-4 bg-gray-200 rounded w-16 mx-auto animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4"><div className="h-4 bg-gray-200 rounded w-14 ml-auto animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4"><div className="h-4 bg-gray-200 rounded w-10 ml-auto animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4"><div className="h-4 bg-gray-200 rounded w-10 ml-auto animate-pulse"></div></td>
|
||||||
|
<td className="py-3 px-4 text-center"><div className="h-4 bg-gray-200 rounded w-8 mx-auto animate-pulse"></div></td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
) : history.length === 0 ? (
|
||||||
|
<div className="text-center py-12">
|
||||||
|
<Clock className="w-12 h-12 text-gray-400 mx-auto mb-3" />
|
||||||
|
<p className="text-gray-600">No scraper runs recorded yet.</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full min-w-max">
|
||||||
|
<thead className="bg-gray-50 border-b">
|
||||||
|
<tr>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700">Time</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Status</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Trigger</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Duration</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Units</th>
|
||||||
|
<th className="text-right py-3 px-4 font-semibold text-gray-700">Prices</th>
|
||||||
|
<th className="text-center py-3 px-4 font-semibold text-gray-700">Dry Run</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{history.map((run, index) => (
|
||||||
|
<tr key={run._id || index} className={`border-b ${index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}`}>
|
||||||
|
<td className="py-3 px-4 text-sm text-gray-600">
|
||||||
|
{formatDate(run.startedAt)}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-center">
|
||||||
|
<span className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium ${
|
||||||
|
run.status === 'success'
|
||||||
|
? 'bg-green-100 text-green-800'
|
||||||
|
: run.status === 'running'
|
||||||
|
? 'bg-amber-100 text-amber-800'
|
||||||
|
: 'bg-red-100 text-red-800'
|
||||||
|
}`}>
|
||||||
|
{run.status === 'success' ? 'Success' : run.status === 'running' ? 'Running' : 'Failed'}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-center text-sm text-gray-600 capitalize">
|
||||||
|
{run.trigger || 'N/A'}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-right text-sm text-gray-600">
|
||||||
|
{formatDuration(run.duration)}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-right text-sm text-gray-600">
|
||||||
|
{run.unitsProcessed ?? '-'}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-right text-sm text-gray-600">
|
||||||
|
{run.pricesInserted ?? '-'}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-center text-sm text-gray-600">
|
||||||
|
{run.dryRun ? 'Yes' : 'No'}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user