diff --git a/CHANGELOG.md b/CHANGELOG.md index e673051..f1077af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.30.0] - 2026-05-20 + +### Changed +- The "+ New" menu now also creates **Kafka topic**, **URL resource** and **S3 resource** (next to Organization, Dataset, Service). Their pages (`/kafka-topics`, `/url-resources`, `/s3-resources`) are simplified to single-purpose creation forms — listing, editing and deletion are dropped, mirroring the earlier organization/service/dataset reorganization. Listing and deletion of these resources now happen on the Search page (they are CKAN packages, appear in the Datasets group, and owned ones expose the existing Delete action). +- The "Resources" navigation dropdown is removed. Its only remaining entry that is not a catalog-creation flow — **S3 Management** (bucket/object management) — is now a top-level navigation item. + +### Backwards compatibility +- UI-only reorganization. No API behavior, request/response shapes, or routes change. The `/kafka-topics`, `/url-resources`, `/s3-resources` routes remain (now create-only) and `/s3-management` is unchanged. +- Resources registered before the creator-hash feature do not expose a Delete action on Search; no migration is performed. + ## [0.29.0] - 2026-05-19 ### Added diff --git a/api/config/swagger_settings.py b/api/config/swagger_settings.py index e90c11b..9811dfe 100644 --- a/api/config/swagger_settings.py +++ b/api/config/swagger_settings.py @@ -12,7 +12,7 @@ class Settings(BaseSettings): swagger_title: str = "API Documentation" swagger_description: str = "This is the API documentation." - swagger_version: str = "0.29.0" + swagger_version: str = "0.30.0" root_path: str = "" # API root path prefix (e.g., "/test" or "") is_public: bool = True metrics_endpoint: str = "https://federation.ndp.utah.edu/metrics/" diff --git a/ui/src/components/Navigation.js b/ui/src/components/Navigation.js index 6debeaf..1d87f43 100644 --- a/ui/src/components/Navigation.js +++ b/ui/src/components/Navigation.js @@ -10,7 +10,6 @@ import { Search, FileText, LogOut, - FolderOpen, ChevronDown, HardDrive, ShieldAlert, @@ -23,10 +22,6 @@ import { isAccessRequestAdmin, userAPI } from '../services/api'; * Gray background with improved dropdown logic that always closes properly */ const Navigation = () => { - const [isDropdownOpen, setIsDropdownOpen] = useState(false); - const dropdownRef = useRef(null); - const buttonRef = useRef(null); - const timeoutRef = useRef(null); const [isNewMenuOpen, setIsNewMenuOpen] = useState(false); const newMenuRef = useRef(null); const newButtonRef = useRef(null); @@ -60,25 +55,16 @@ const Navigation = () => { }; }, []); - // Clear timeouts on unmount + // Clear timeout on unmount useEffect(() => { return () => { - if (timeoutRef.current) clearTimeout(timeoutRef.current); if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); }; }, []); - // Close dropdowns when clicking outside + // Close the "+ New" menu when clicking outside useEffect(() => { const handleClickOutside = (event) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target) && - buttonRef.current && - !buttonRef.current.contains(event.target) - ) { - setIsDropdownOpen(false); - } if ( newMenuRef.current && !newMenuRef.current.contains(event.target) && @@ -95,26 +81,9 @@ const Navigation = () => { }; }, []); - // Improved dropdown handlers with timeout for better UX - const handleDropdownEnter = () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current); - } - setIsDropdownOpen(true); - setIsNewMenuOpen(false); - }; - - const handleDropdownLeave = () => { - // Add small delay to prevent accidental closes - timeoutRef.current = setTimeout(() => { - setIsDropdownOpen(false); - }, 150); - }; - const handleNewMenuEnter = () => { if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); setIsNewMenuOpen(true); - setIsDropdownOpen(false); }; const handleNewMenuLeave = () => { @@ -123,11 +92,9 @@ const Navigation = () => { }, 150); }; - // Handle mouse entering other nav items - close dropdowns immediately + // Handle mouse entering other nav items - close the menu immediately const handleOtherNavEnter = () => { - if (timeoutRef.current) clearTimeout(timeoutRef.current); if (newTimeoutRef.current) clearTimeout(newTimeoutRef.current); - setIsDropdownOpen(false); setIsNewMenuOpen(false); }; @@ -210,52 +177,86 @@ const Navigation = () => { Search - {/* Resources Dropdown */} -
{ + e.target.style.color = '#374151'; + e.target.style.fontWeight = '600'; + }} + onMouseOut={(e) => { + e.target.style.color = '#6b7280'; + e.target.style.fontWeight = '500'; + }} + > + + S3 Management + + + {/* + New menu — only visible to users that can write */} + {canWrite && ( +
- {/* Dropdown */} - {isDropdownOpen && ( + {isNewMenuOpen && (
{ }} > { e.target.style.fontWeight = '500'; }} > - - Kafka Topics + + Organization { e.target.style.fontWeight = '500'; }} > - - URL Resources + + Dataset { e.target.style.fontWeight = '500'; }} > - - S3 Resources - - - { - e.target.style.backgroundColor = '#f9fafb'; - e.target.style.color = '#2563eb'; - e.target.style.fontWeight = '600'; - }} - onMouseOut={(e) => { - e.target.style.backgroundColor = 'white'; - e.target.style.color = '#374151'; - e.target.style.fontWeight = '500'; - }} - > - - S3 Management + + Service -
- )} -
- - {/* + New menu — only visible to users that can write */} - {canWrite && ( -
- - {isNewMenuOpen && ( -
{ e.target.style.fontWeight = '500'; }} > - - Organization + + Kafka topic { e.target.style.fontWeight = '500'; }} > - - Dataset + + URL resource { e.target.style.fontWeight = '500'; }} > - - Service + + S3 resource
)} diff --git a/ui/src/pages/KafkaTopics.js b/ui/src/pages/KafkaTopics.js index 89d49a2..cd9b798 100644 --- a/ui/src/pages/KafkaTopics.js +++ b/ui/src/pages/KafkaTopics.js @@ -1,471 +1,196 @@ -import React, { useState, useEffect, useCallback } from 'react'; -import { - Radio, - Plus, - AlertCircle, - Edit3, - Save, - X, - Trash2, - RefreshCw, - Database -} from 'lucide-react'; -import { kafkaAPI, organizationsAPI, searchAPI, resourcesAPI } from '../services/api'; +import React, { useEffect, useState } from 'react'; +import { Radio, AlertCircle, CheckCircle, Plus, Trash2 } from 'lucide-react'; +import { useNavigate } from 'react-router-dom'; +import { kafkaAPI, organizationsAPI } from '../services/api'; /** - * Kafka Topics page component for managing Kafka data sources - * Allows creating, listing, editing, and deleting Kafka topic datasets + * Single-purpose page reached from "+ New > Kafka topic". Registers a + * Kafka topic as a catalog dataset. Listing and deletion live on the + * Search page (Kafka topics are CKAN packages and show up there). */ const KafkaTopics = () => { - const [kafkaTopics, setKafkaTopics] = useState([]); - const [organizations, setOrganizations] = useState([]); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - const [success, setSuccess] = useState(null); - const [showCreateForm, setShowCreateForm] = useState(false); - const [editingTopic, setEditingTopic] = useState(null); - const [selectedServer] = useState('local'); // Fixed to local for consistency - - // Form state for creating/editing Kafka topic + const navigate = useNavigate(); const [formData, setFormData] = useState({ dataset_name: '', dataset_title: '', owner_org: '', + dataset_description: '', kafka_topic: '', kafka_host: '', - kafka_port: '', - dataset_description: '', - extras: {}, - mapping: {}, - processing: {} + kafka_port: '' }); + const [extrasPairs, setExtrasPairs] = useState([]); + const [organizations, setOrganizations] = useState([]); + const [orgsLoading, setOrgsLoading] = useState(true); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(null); - // JSON editor states for complex fields - const [extrasJson, setExtrasJson] = useState('{}'); - const [mappingJson, setMappingJson] = useState('{}'); - const [processingJson, setProcessingJson] = useState('{}'); - - /** - * Fetch organizations for dropdown - */ - const fetchOrganizations = useCallback(async () => { - try { - const response = await organizationsAPI.list({ server: selectedServer }); - setOrganizations(response.data || []); - } catch (err) { - console.error('Error fetching organizations:', err); - } - }, [selectedServer]); - - /** - * Fetch all Kafka topics (filtered from all datasets) - */ - const fetchKafkaTopics = useCallback(async () => { - try { - setLoading(true); - setError(null); - - // Use search to get all datasets, then filter for Kafka topics - const response = await searchAPI.searchByTerms([''], null, selectedServer); - - // Filter to show only Kafka topics - const filteredTopics = (response.data || []).filter(dataset => { - const extras = dataset.extras || {}; - - // Include if it has Kafka-specific metadata - // Check for both old format (kafka_*) and new format (host, port, topic) - return ( - extras.kafka_topic || extras.kafka_host || extras.kafka_port || // Old format - extras.topic || extras.host || extras.port || // New format - // Also check if any resource has format "kafka" - (dataset.resources && dataset.resources.some(resource => - resource.format === 'kafka' - )) + useEffect(() => { + let cancelled = false; + organizationsAPI + .list({ server: 'local' }) + .then((response) => { + if (cancelled) return; + const orgs = Array.isArray(response.data) ? response.data : []; + setOrganizations(orgs); + setFormData((prev) => + prev.owner_org ? prev : { ...prev, owner_org: orgs[0] || '' } ); + }) + .catch(() => { + if (!cancelled) setOrganizations([]); + }) + .finally(() => { + if (!cancelled) setOrgsLoading(false); }); - - console.log('All datasets:', response.data); // Debug - console.log('Filtered Kafka topics:', filteredTopics); // Debug - - setKafkaTopics(filteredTopics); - - } catch (err) { - console.error('Error fetching Kafka topics:', err); - setError('Failed to load Kafka topics: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }, [selectedServer]); - - /** - * Fetch organizations and Kafka topics on component mount - */ - useEffect(() => { - fetchOrganizations(); - fetchKafkaTopics(); - }, [fetchOrganizations, fetchKafkaTopics]); + return () => { + cancelled = true; + }; + }, []); - /** - * Handle form input changes - */ const handleInputChange = (e) => { const { name, value } = e.target; - setFormData(prev => ({ - ...prev, - [name]: value - })); + setFormData((prev) => ({ ...prev, [name]: value })); }; - /** - * Parse JSON string safely - */ - const parseJsonSafely = (jsonString, fallback = {}) => { - try { - return jsonString.trim() === '' ? fallback : JSON.parse(jsonString); - } catch { - return fallback; + const addExtraPair = () => + setExtrasPairs((prev) => [...prev, { key: '', value: '' }]); + const removeExtraPair = (idx) => + setExtrasPairs((prev) => prev.filter((_, i) => i !== idx)); + const updateExtraPair = (idx, field, value) => + setExtrasPairs((prev) => + prev.map((p, i) => (i === idx ? { ...p, [field]: value } : p)) + ); + + const buildExtras = () => { + const out = {}; + for (const { key, value } of extrasPairs) { + const trimmed = (key || '').trim(); + if (trimmed) out[trimmed] = value; } + return out; }; - /** - * Reset form to initial state - */ - const resetForm = () => { - setFormData({ - dataset_name: '', - dataset_title: '', - owner_org: '', - kafka_topic: '', - kafka_host: '', - kafka_port: '', - dataset_description: '', - extras: {}, - mapping: {}, - processing: {} - }); - setExtrasJson('{}'); - setMappingJson('{}'); - setProcessingJson('{}'); - setEditingTopic(null); - setShowCreateForm(false); - }; - - /** - * Prepare form data for submission - */ - const prepareFormData = () => { - // Parse JSON fields - const extras = parseJsonSafely(extrasJson, {}); - const mapping = parseJsonSafely(mappingJson, {}); - const processing = parseJsonSafely(processingJson, {}); - - // Prepare data - const requestData = { - ...formData, - extras: Object.keys(extras).length > 0 ? extras : undefined, - mapping: Object.keys(mapping).length > 0 ? mapping : undefined, - processing: Object.keys(processing).length > 0 ? processing : undefined - }; - - // Remove empty fields - Object.keys(requestData).forEach(key => { - if (requestData[key] === '' || requestData[key] === undefined) { - delete requestData[key]; - } - }); - - return requestData; - }; - - /** - * Handle form submission for creating Kafka topic - */ - const handleCreate = async (e) => { + const handleSubmit = async (e) => { e.preventDefault(); - - try { - setError(null); - setSuccess(null); - setLoading(true); - - const requestData = prepareFormData(); - await kafkaAPI.create(requestData, selectedServer); - - setSuccess('Kafka topic dataset created successfully!'); - resetForm(); - fetchKafkaTopics(); - - } catch (err) { - console.error('Error creating Kafka topic:', err); - setError('Failed to create Kafka topic: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }; - - /** - * Handle form submission for updating Kafka topic - */ - const handleUpdate = async (e) => { - e.preventDefault(); - - try { - setError(null); - setSuccess(null); - setLoading(true); - - const requestData = prepareFormData(); - await kafkaAPI.update(editingTopic.id, requestData, selectedServer); - - setSuccess('Kafka topic updated successfully!'); - resetForm(); - fetchKafkaTopics(); - - } catch (err) { - console.error('Error updating Kafka topic:', err); - setError('Failed to update Kafka topic: ' + - (err.response?.data?.detail || err.message)); - } finally { - setLoading(false); - } - }; - - /** - * Start editing a Kafka topic - * FIXED: Properly separate Kafka-specific fields from general extras - */ - const startEditing = (topic) => { - setEditingTopic(topic); - const extras = topic.extras || {}; - - // Extract Kafka-specific fields from extras - const kafkaFields = { - kafka_topic: extras.kafka_topic || extras.topic || '', - kafka_host: extras.kafka_host || extras.host || '', - kafka_port: extras.kafka_port || extras.port || '' + setError(null); + setSuccess(null); + setSubmitting(true); + + const payload = { + dataset_name: formData.dataset_name, + dataset_title: formData.dataset_title, + owner_org: formData.owner_org, + kafka_topic: formData.kafka_topic, + kafka_host: formData.kafka_host, + kafka_port: Number(formData.kafka_port) }; - - // Create a clean extras object without the reserved Kafka fields - const cleanExtras = { ...extras }; - - // Remove reserved Kafka fields from extras - const reservedKafkaFields = [ - 'kafka_topic', 'topic', - 'kafka_host', 'host', - 'kafka_port', 'port' - ]; - - reservedKafkaFields.forEach(field => { - delete cleanExtras[field]; - }); - - // Also extract mapping and processing from extras if they exist - const mapping = cleanExtras.mapping || {}; - const processing = cleanExtras.processing || {}; - - // Remove mapping and processing from cleanExtras since they have their own fields - delete cleanExtras.mapping; - delete cleanExtras.processing; - - setFormData({ - dataset_name: topic.name || '', - dataset_title: topic.title || '', - owner_org: topic.owner_org || '', - kafka_topic: kafkaFields.kafka_topic, - kafka_host: kafkaFields.kafka_host, - kafka_port: kafkaFields.kafka_port, - dataset_description: topic.notes || '', - extras: cleanExtras, - mapping: mapping, - processing: processing - }); - - // Set JSON fields with clean data - setExtrasJson(JSON.stringify(cleanExtras, null, 2)); - setMappingJson(JSON.stringify(mapping, null, 2)); - setProcessingJson(JSON.stringify(processing, null, 2)); - setShowCreateForm(true); - }; - - /** - * Handle Kafka topic deletion - */ - const handleDeleteTopic = async (topic) => { - const displayName = topic.title || topic.name || 'Unnamed Topic'; - - if (!window.confirm( - `Are you sure you want to delete Kafka topic "${displayName}"? This will also delete all associated resources. This action cannot be undone.` - )) { - return; - } + if (formData.dataset_description) + payload.dataset_description = formData.dataset_description; + const extras = buildExtras(); + if (Object.keys(extras).length > 0) payload.extras = extras; try { - setError(null); - setSuccess(null); - - // Debug: Log the topic being deleted - console.log('Attempting to delete Kafka topic with ID:', topic.id); - console.log('Using endpoint: DELETE /resource?resource_id=' + topic.id + '&server=local'); - - // Use the resource deletion endpoint since datasets are resources in CKAN - await resourcesAPI.deleteById(topic.id, 'local'); - - setSuccess(`Kafka topic "${displayName}" deleted successfully!`); - - // Refresh the topics list - fetchKafkaTopics(); - + await kafkaAPI.create(payload, 'local'); + setSuccess( + `Kafka topic "${formData.dataset_name}" registered. You can keep registering more or go back to Search.` + ); + setFormData((prev) => ({ + ...prev, + dataset_name: '', + dataset_title: '', + dataset_description: '', + kafka_topic: '', + kafka_host: '', + kafka_port: '' + })); + setExtrasPairs([]); } catch (err) { - console.error('Error deleting Kafka topic:', err); - console.error('Full error object:', err); - console.error('Error response:', err.response); - - let errorMessage = 'Failed to delete Kafka topic: '; - - if (err.response?.status === 404) { - errorMessage += `Topic "${displayName}" not found. It may have been already deleted.`; - } else if (err.response?.status === 405) { - errorMessage += 'Topic deletion method not allowed.'; - } else if (err.response?.status === 401) { - errorMessage += 'Authentication required. Please login again.'; - } else if (err.response?.status === 403) { - errorMessage += 'You do not have permission to delete this topic.'; - } else { - errorMessage += (err.response?.data?.detail || err.message); - } - - setError(errorMessage); - } - }; - - /** - * Get connection status badge - */ - const getConnectionStatus = (topic) => { - const extras = topic.extras || {}; - // Check both old format (kafka_*) and new format (host, port, topic) - const hasHost = extras.kafka_host || extras.host; - const hasPort = extras.kafka_port || extras.port; - const hasTopic = extras.kafka_topic || extras.topic; - - if (hasHost && hasPort && hasTopic) { - return { status: 'Connected', color: 'status-success' }; + const detail = err.response?.data?.detail; + const raw = typeof detail === 'string' ? detail : err.message; + const msg = raw || 'unknown error'; + setError( + /already exists/i.test(msg) + ? `A dataset called "${formData.dataset_name}" already exists. Pick a different name.` + : `Could not register the Kafka topic: ${msg}.` + ); + } finally { + setSubmitting(false); } - return { status: 'Incomplete', color: 'status-warning' }; }; return ( -
- {/* Page Header */} +

- Kafka Topics + New Kafka topic

- Register and manage Kafka topics as datasets in your CKAN instance + Register a Kafka topic as a catalog dataset so others can + discover the stream and how to connect to it. Find it (and + remove it) from the Search page once created.

- {/* Alert Messages */} {error && (
{error}
)} - {success && (
+ {success}
)} - {/* Page actions */} -
- - {!showCreateForm && ( - - )} -
+
+
+
+ + + + Unique identifier — lowercase letters, numbers, underscores + and hyphens only. + +
- {/* Create/Edit Kafka Topic Form */} - {showCreateForm && ( -
-
-

- {editingTopic ? ( - <> - - Edit Kafka Topic: {editingTopic.title} - - ) : ( - <> - - Create New Kafka Topic Dataset - - )} -

- +
+ +
- - {/* Basic Information */} -
-
- - - - Unique identifier for the dataset - +
+ + {orgsLoading ? ( +
+ Loading organizations…
- -
- - + ) : organizations.length === 0 ? ( +
+ No organizations available. Create one from "+ New → + Organization" first.
-
- -
- + ) : ( -
- - {/* Kafka Configuration */} -
-
- - -
+ )} +
-
- - -
+
+ +