From 697168e721dd045243a206840b37b6a1fb336c5b Mon Sep 17 00:00:00 2001 From: Raul Bardaji Date: Thu, 21 May 2026 02:49:09 -0600 Subject: [PATCH 1/2] feat(ui): move Kafka/URL/S3 resource creation into the + New menu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #175. Mirror the organization/service/dataset reorganization for the three remaining catalog resource types: - The "+ New" menu gains "Kafka topic", "URL resource" and "S3 resource" items next to Organization, Dataset and Service. - /kafka-topics, /url-resources and /s3-resources are simplified to single-purpose creation forms (org dropdown + type-specific fields + key/value extras). Listing, editing and deletion are dropped — those resources are CKAN packages, so they already appear in the Search Datasets group and owned ones expose the existing Delete action. - The "Resources" navigation dropdown is removed. Its only remaining entry that is not a catalog-creation flow, S3 Management (bucket and object management), becomes a top-level nav item. The now-unused dropdown state/refs/handlers and the FolderOpen import are cleaned up. Routes are unchanged (/kafka-topics, /url-resources, /s3-resources stay, now create-only; /s3-management unchanged), and no API behavior or payload shapes change. --- ui/src/components/Navigation.js | 251 +++----- ui/src/pages/KafkaTopics.js | 896 ++++++++------------------ ui/src/pages/S3Resources.js | 859 +++++++------------------ ui/src/pages/UrlResources.js | 1046 +++++++------------------------ 4 files changed, 780 insertions(+), 2272 deletions(-) diff --git a/ui/src/components/Navigation.js b/ui/src/components/Navigation.js index 6debeaf..1d87f43 100644 --- a/ui/src/components/Navigation.js +++ b/ui/src/components/Navigation.js @@ -10,7 +10,6 @@ import { Search, FileText, LogOut, - FolderOpen, ChevronDown, HardDrive, ShieldAlert, @@ -23,10 +22,6 @@ import { isAccessRequestAdmin, userAPI } from '../services/api'; * Gray background with improved dropdown logic that always closes properly */ const Navigation = () => { - const [isDropdownOpen, setIsDropdownOpen] = useState(false); - const dropdownRef = useRef(null); - const buttonRef = useRef(null); - const timeoutRef = useRef(null); const [isNewMenuOpen, setIsNewMenuOpen] = useState(false); const newMenuRef = useRef(null); const newButtonRef = useRef(null); @@ -60,25 +55,16 @@ const Navigation = () => { }; }, []); - // Clear timeouts on unmount + // Clear timeout on unmount useEffect(() => { return () => { - if (timeoutRef.current) clearTimeout(timeoutRef.current); if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); }; }, []); - // Close dropdowns when clicking outside + // Close the "+ New" menu when clicking outside useEffect(() => { const handleClickOutside = (event) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target) && - buttonRef.current && - !buttonRef.current.contains(event.target) - ) { - setIsDropdownOpen(false); - } if ( newMenuRef.current && !newMenuRef.current.contains(event.target) && @@ -95,26 +81,9 @@ const Navigation = () => { }; }, []); - // Improved dropdown handlers with timeout for better UX - const handleDropdownEnter = () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current); - } - setIsDropdownOpen(true); - setIsNewMenuOpen(false); - }; - - const handleDropdownLeave = () => { - // Add small delay to prevent accidental closes - timeoutRef.current = setTimeout(() => { - setIsDropdownOpen(false); - }, 150); - }; - const handleNewMenuEnter = () => { if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); setIsNewMenuOpen(true); - setIsDropdownOpen(false); }; const handleNewMenuLeave = () => { @@ -123,11 +92,9 @@ const Navigation = () => { }, 150); }; - // Handle mouse entering other nav items - close dropdowns immediately + // Handle mouse entering other nav items - close the menu immediately const handleOtherNavEnter = () => { - if (timeoutRef.current) clearTimeout(timeoutRef.current); if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); - setIsDropdownOpen(false); setIsNewMenuOpen(false); }; @@ -210,52 +177,86 @@ const Navigation = () => { Search - {/* Resources Dropdown */} -
{ + e.target.style.color = '#374151'; + e.target.style.fontWeight = '600'; + }} + onMouseOut={(e) => { + e.target.style.color = '#6b7280'; + e.target.style.fontWeight = '500'; + }} + > + + S3 Management + + + {/* + New menu — only visible to users that can write */} + {canWrite && ( +
- {/* Dropdown */} - {isDropdownOpen && ( + {isNewMenuOpen && (
{ }} > { e.target.style.fontWeight = '500'; }} > - - Kafka Topics + + Organization { e.target.style.fontWeight = '500'; }} > - - URL Resources + + Dataset { e.target.style.fontWeight = '500'; }} > - - S3 Resources - - - { - e.target.style.backgroundColor = '#f9fafb'; - e.target.style.color = '#2563eb'; - e.target.style.fontWeight = '600'; - }} - onMouseOut={(e) => { - e.target.style.backgroundColor = 'white'; - e.target.style.color = '#374151'; - e.target.style.fontWeight = '500'; - }} - > - - S3 Management + + Service -
- )} -
- - {/* + New menu — only visible to users that can write */} - {canWrite && ( -
- - {isNewMenuOpen && ( -
{ e.target.style.fontWeight = '500'; }} > - - Organization + + Kafka topic { e.target.style.fontWeight = '500'; }} > - - Dataset + + URL resource { e.target.style.fontWeight = '500'; }} > - - Service + + S3 resource
)} diff --git a/ui/src/pages/KafkaTopics.js b/ui/src/pages/KafkaTopics.js index 89d49a2..cd9b798 100644 --- a/ui/src/pages/KafkaTopics.js +++ b/ui/src/pages/KafkaTopics.js @@ -1,471 +1,196 @@ -import React, { useState, useEffect, useCallback } from 'react'; -import { - Radio, - Plus, - AlertCircle, - Edit3, - Save, - X, - Trash2, - RefreshCw, - Database -} from 'lucide-react'; -import { kafkaAPI, organizationsAPI, searchAPI, resourcesAPI } from '../services/api'; +import React, { useEffect, useState } from 'react'; +import { Radio, AlertCircle, CheckCircle, Plus, Trash2 } from 'lucide-react'; +import { useNavigate } from 'react-router-dom'; +import { kafkaAPI, organizationsAPI } from '../services/api'; /** - * Kafka Topics page component for managing Kafka data sources - * Allows creating, listing, editing, and deleting Kafka topic datasets + * Single-purpose page reached from "+ New > Kafka topic". Registers a + * Kafka topic as a catalog dataset. Listing and deletion live on the + * Search page (Kafka topics are CKAN packages and show up there). */ const KafkaTopics = () => { - const [kafkaTopics, setKafkaTopics] = useState([]); - const [organizations, setOrganizations] = useState([]); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - const [success, setSuccess] = useState(null); - const [showCreateForm, setShowCreateForm] = useState(false); - const [editingTopic, setEditingTopic] = useState(null); - const [selectedServer] = useState('local'); // Fixed to local for consistency - - // Form state for creating/editing Kafka topic + const navigate = useNavigate(); const [formData, setFormData] = useState({ dataset_name: '', dataset_title: '', owner_org: '', + dataset_description: '', kafka_topic: '', kafka_host: '', - kafka_port: '', - dataset_description: '', - extras: {}, - mapping: {}, - processing: {} + kafka_port: '' }); + const [extrasPairs, setExtrasPairs] = useState([]); + const [organizations, setOrganizations] = useState([]); + const [orgsLoading, setOrgsLoading] = useState(true); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(null); - // JSON editor states for complex fields - const [extrasJson, setExtrasJson] = useState('{}'); - const [mappingJson, setMappingJson] = useState('{}'); - const [processingJson, setProcessingJson] = useState('{}'); - - /** - * Fetch organizations for dropdown - */ - const fetchOrganizations = useCallback(async () => { - try { - const response = await organizationsAPI.list({ server: selectedServer }); - setOrganizations(response.data || []); - } catch (err) { - console.error('Error fetching organizations:', err); - } - }, [selectedServer]); - - /** - * Fetch all Kafka topics (filtered from all datasets) - */ - const fetchKafkaTopics = useCallback(async () => { - try { - setLoading(true); - setError(null); - - // Use search to get all datasets, then filter for Kafka topics - const response = await searchAPI.searchByTerms([''], null, selectedServer); - - // Filter to show only Kafka topics - const filteredTopics = (response.data || []).filter(dataset => { - const extras = dataset.extras || {}; - - // Include if it has Kafka-specific metadata - // Check for both old format (kafka_*) and new format (host, port, topic) - return ( - extras.kafka_topic || extras.kafka_host || extras.kafka_port || // Old format - extras.topic || extras.host || extras.port || // New format - // Also check if any resource has format "kafka" - (dataset.resources && dataset.resources.some(resource => - resource.format === 'kafka' - )) + useEffect(() => { + let cancelled = false; + organizationsAPI + .list({ server: 'local' }) + .then((response) => { + if (cancelled) return; + const orgs = Array.isArray(response.data) ? response.data : []; + setOrganizations(orgs); + setFormData((prev) => + prev.owner_org ? prev : { ...prev, owner_org: orgs[0] || '' } ); + }) + .catch(() => { + if (!cancelled) setOrganizations([]); + }) + .finally(() => { + if (!cancelled) setOrgsLoading(false); }); - - console.log('All datasets:', response.data); // Debug - console.log('Filtered Kafka topics:', filteredTopics); // Debug - - setKafkaTopics(filteredTopics); - - } catch (err) { - console.error('Error fetching Kafka topics:', err); - setError('Failed to load Kafka topics: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }, [selectedServer]); - - /** - * Fetch organizations and Kafka topics on component mount - */ - useEffect(() => { - fetchOrganizations(); - fetchKafkaTopics(); - }, [fetchOrganizations, fetchKafkaTopics]); + return () => { + cancelled = true; + }; + }, []); - /** - * Handle form input changes - */ const handleInputChange = (e) => { const { name, value } = e.target; - setFormData(prev => ({ - ...prev, - [name]: value - })); + setFormData((prev) => ({ ...prev, [name]: value })); }; - /** - * Parse JSON string safely - */ - const parseJsonSafely = (jsonString, fallback = {}) => { - try { - return jsonString.trim() === '' ? fallback : JSON.parse(jsonString); - } catch { - return fallback; + const addExtraPair = () => + setExtrasPairs((prev) => [...prev, { key: '', value: '' }]); + const removeExtraPair = (idx) => + setExtrasPairs((prev) => prev.filter((_, i) => i !== idx)); + const updateExtraPair = (idx, field, value) => + setExtrasPairs((prev) => + prev.map((p, i) => (i === idx ? { ...p, [field]: value } : p)) + ); + + const buildExtras = () => { + const out = {}; + for (const { key, value } of extrasPairs) { + const trimmed = (key || '').trim(); + if (trimmed) out[trimmed] = value; } + return out; }; - /** - * Reset form to initial state - */ - const resetForm = () => { - setFormData({ - dataset_name: '', - dataset_title: '', - owner_org: '', - kafka_topic: '', - kafka_host: '', - kafka_port: '', - dataset_description: '', - extras: {}, - mapping: {}, - processing: {} - }); - setExtrasJson('{}'); - setMappingJson('{}'); - setProcessingJson('{}'); - setEditingTopic(null); - setShowCreateForm(false); - }; - - /** - * Prepare form data for submission - */ - const prepareFormData = () => { - // Parse JSON fields - const extras = parseJsonSafely(extrasJson, {}); - const mapping = parseJsonSafely(mappingJson, {}); - const processing = parseJsonSafely(processingJson, {}); - - // Prepare data - const requestData = { - ...formData, - extras: Object.keys(extras).length > 0 ? extras : undefined, - mapping: Object.keys(mapping).length > 0 ? mapping : undefined, - processing: Object.keys(processing).length > 0 ? processing : undefined - }; - - // Remove empty fields - Object.keys(requestData).forEach(key => { - if (requestData[key] === '' || requestData[key] === undefined) { - delete requestData[key]; - } - }); - - return requestData; - }; - - /** - * Handle form submission for creating Kafka topic - */ - const handleCreate = async (e) => { + const handleSubmit = async (e) => { e.preventDefault(); - - try { - setError(null); - setSuccess(null); - setLoading(true); - - const requestData = prepareFormData(); - await kafkaAPI.create(requestData, selectedServer); - - setSuccess('Kafka topic dataset created successfully!'); - resetForm(); - fetchKafkaTopics(); - - } catch (err) { - console.error('Error creating Kafka topic:', err); - setError('Failed to create Kafka topic: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }; - - /** - * Handle form submission for updating Kafka topic - */ - const handleUpdate = async (e) => { - e.preventDefault(); - - try { - setError(null); - setSuccess(null); - setLoading(true); - - const requestData = prepareFormData(); - await kafkaAPI.update(editingTopic.id, requestData, selectedServer); - - setSuccess('Kafka topic updated successfully!'); - resetForm(); - fetchKafkaTopics(); - - } catch (err) { - console.error('Error updating Kafka topic:', err); - setError('Failed to update Kafka topic: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }; - - /** - * Start editing a Kafka topic - * FIXED: Properly separate Kafka-specific fields from general extras - */ - const startEditing = (topic) => { - setEditingTopic(topic); - const extras = topic.extras || {}; - - // Extract Kafka-specific fields from extras - const kafkaFields = { - kafka_topic: extras.kafka_topic || extras.topic || '', - kafka_host: extras.kafka_host || extras.host || '', - kafka_port: extras.kafka_port || extras.port || '' + setError(null); + setSuccess(null); + setSubmitting(true); + + const payload = { + dataset_name: formData.dataset_name, + dataset_title: formData.dataset_title, + owner_org: formData.owner_org, + kafka_topic: formData.kafka_topic, + kafka_host: formData.kafka_host, + kafka_port: Number(formData.kafka_port) }; - - // Create a clean extras object without the reserved Kafka fields - const cleanExtras = { ...extras }; - - // Remove reserved Kafka fields from extras - const reservedKafkaFields = [ - 'kafka_topic', 'topic', - 'kafka_host', 'host', - 'kafka_port', 'port' - ]; - - reservedKafkaFields.forEach(field => { - delete cleanExtras[field]; - }); - - // Also extract mapping and processing from extras if they exist - const mapping = cleanExtras.mapping || {}; - const processing = cleanExtras.processing || {}; - - // Remove mapping and processing from cleanExtras since they have their own fields - delete cleanExtras.mapping; - delete cleanExtras.processing; - - setFormData({ - dataset_name: topic.name || '', - dataset_title: topic.title || '', - owner_org: topic.owner_org || '', - kafka_topic: kafkaFields.kafka_topic, - kafka_host: kafkaFields.kafka_host, - kafka_port: kafkaFields.kafka_port, - dataset_description: topic.notes || '', - extras: cleanExtras, - mapping: mapping, - processing: processing - }); - - // Set JSON fields with clean data - setExtrasJson(JSON.stringify(cleanExtras, null, 2)); - setMappingJson(JSON.stringify(mapping, null, 2)); - setProcessingJson(JSON.stringify(processing, null, 2)); - setShowCreateForm(true); - }; - - /** - * Handle Kafka topic deletion - */ - const handleDeleteTopic = async (topic) => { - const displayName = topic.title || topic.name || 'Unnamed Topic'; - - if (!window.confirm( - `Are you sure you want to delete Kafka topic "${displayName}"? This will also delete all associated resources. This action cannot be undone.` - )) { - return; - } + if (formData.dataset_description) + payload.dataset_description = formData.dataset_description; + const extras = buildExtras(); + if (Object.keys(extras).length > 0) payload.extras = extras; try { - setError(null); - setSuccess(null); - - // Debug: Log the topic being deleted - console.log('Attempting to delete Kafka topic with ID:', topic.id); - console.log('Using endpoint: DELETE /resource?resource_id=' + topic.id + '&server=local'); - - // Use the resource deletion endpoint since datasets are resources in CKAN - await resourcesAPI.deleteById(topic.id, 'local'); - - setSuccess(`Kafka topic "${displayName}" deleted successfully!`); - - // Refresh the topics list - fetchKafkaTopics(); - + await kafkaAPI.create(payload, 'local'); + setSuccess( + `Kafka topic "${formData.dataset_name}" registered. You can keep registering more or go back to Search.` + ); + setFormData((prev) => ({ + ...prev, + dataset_name: '', + dataset_title: '', + dataset_description: '', + kafka_topic: '', + kafka_host: '', + kafka_port: '' + })); + setExtrasPairs([]); } catch (err) { - console.error('Error deleting Kafka topic:', err); - console.error('Full error object:', err); - console.error('Error response:', err.response); - - let errorMessage = 'Failed to delete Kafka topic: '; - - if (err.response?.status === 404) { - errorMessage += `Topic "${displayName}" not found. It may have been already deleted.`; - } else if (err.response?.status === 405) { - errorMessage += 'Topic deletion method not allowed.'; - } else if (err.response?.status === 401) { - errorMessage += 'Authentication required. Please login again.'; - } else if (err.response?.status === 403) { - errorMessage += 'You do not have permission to delete this topic.'; - } else { - errorMessage += (err.response?.data?.detail || err.message); - } - - setError(errorMessage); - } - }; - - /** - * Get connection status badge - */ - const getConnectionStatus = (topic) => { - const extras = topic.extras || {}; - // Check both old format (kafka_*) and new format (host, port, topic) - const hasHost = extras.kafka_host || extras.host; - const hasPort = extras.kafka_port || extras.port; - const hasTopic = extras.kafka_topic || extras.topic; - - if (hasHost && hasPort && hasTopic) { - return { status: 'Connected', color: 'status-success' }; + const detail = err.response?.data?.detail; + const raw = typeof detail === 'string' ? detail : err.message; + const msg = raw || 'unknown error'; + setError( + /already exists/i.test(msg) + ? `A dataset called "${formData.dataset_name}" already exists. Pick a different name.` + : `Could not register the Kafka topic: ${msg}.` + ); + } finally { + setSubmitting(false); } - return { status: 'Incomplete', color: 'status-warning' }; }; return ( -
- {/* Page Header */} +

- Kafka Topics + New Kafka topic

- Register and manage Kafka topics as datasets in your CKAN instance + Register a Kafka topic as a catalog dataset so others can + discover the stream and how to connect to it. Find it (and + remove it) from the Search page once created.

- {/* Alert Messages */} {error && (
{error}
)} - {success && (
+ {success}
)} - {/* Page actions */} -
- - {!showCreateForm && ( - - )} -
+
+
+
+ + + + Unique identifier — lowercase letters, numbers, underscores + and hyphens only. + +
- {/* Create/Edit Kafka Topic Form */} - {showCreateForm && ( -
-
-

- {editingTopic ? ( - <> - - Edit Kafka Topic: {editingTopic.title} - - ) : ( - <> - - Create New Kafka Topic Dataset - - )} -

- +
+ +
- - {/* Basic Information */} -
-
- - - - Unique identifier for the dataset - +
+ + {orgsLoading ? ( +
+ Loading organizations…
- -
- - + ) : organizations.length === 0 ? ( +
+ No organizations available. Create one from "+ New → + Organization" first.
-
- -
- + ) : ( -
- - {/* Kafka Configuration */} -
-
- - -
+ )} +
-
- - -
+
+ +