diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..175d309 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,122 @@ +name: CI/CD Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + test-backend: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Cache Maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Run backend tests + run: | + cd backend + mvn clean test + + - name: Generate test report + uses: dorny/test-reporter@v1 + if: success() || failure() + with: + name: Backend Tests + path: backend/target/surefire-reports/*.xml + reporter: java-junit + + test-frontend: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'npm' + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: | + cd app + npm ci + + - name: Run frontend tests + run: | + cd app + npm run test -- --watchAll=false + + - name: Run linting + run: | + cd app + npm run lint + + build-backend: + needs: test-backend + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Build backend + run: | + cd backend + mvn clean package -DskipTests + + - name: Upload backend artifact + uses: actions/upload-artifact@v3 + with: + name: backend-jar + path: backend/target/*.jar + + build-frontend: + needs: test-frontend + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'npm' + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: | + cd app + npm ci + + - name: Build frontend + run: | + cd app + npm run build + + - name: Upload frontend artifact + uses: actions/upload-artifact@v3 + with: + name: frontend-build + path: app/dist/ \ No newline at end of file diff --git a/.kiro/specs/ml-vector-improvements/.config.kiro b/.kiro/specs/ml-vector-improvements/.config.kiro new file mode 100644 index 0000000..425d49d --- /dev/null +++ b/.kiro/specs/ml-vector-improvements/.config.kiro @@ -0,0 +1 @@ +{"specId": "3c88a2fa-764f-4bbf-a6fe-98bbbe415d5c", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/ml-vector-improvements/design.md b/.kiro/specs/ml-vector-improvements/design.md new file mode 100644 index 0000000..7763ac0 --- /dev/null +++ b/.kiro/specs/ml-vector-improvements/design.md @@ -0,0 +1,515 @@ +# ML Vector Search Improvements - Design Document + +## 1. Architecture Overview + +### System Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ API Layer │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ MyspaceVectorController │ │ +│ │ - search(query, filters, limit) │ │ +│ │ - upsert(items) │ │ +│ │ - health() │ │ +│ └──────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Service Layer │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ VectorSearchService │ │ +│ │ - search with relevance ranking │ │ +│ │ - hybrid search (vector + keyword) │ │ +│ │ - metadata filtering │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ EmbeddingProviderManager │ │ +│ │ - route to configured provider │ │ +│ │ - batch embedding support │ │ +│ │ - error handling & retry logic │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ MetricsCollector │ │ +│ │ - track query latency, cache stats │ │ +│ │ - expose Prometheus metrics │ │ +│ └──────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Provider Layer │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ OpenAIProvider │ │ HuggingFaceProvider │ +│ └──────────────────┘ └──────────────────┘ │ +│ ┌──────────────────┐ │ +│ │ LocalModelProvider │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Cache Layer │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ EmbeddingCache (LRU with TTL) │ │ +│ │ - cache key: hash(text + model_name) │ │ +│ │ - thread-safe concurrent access │ │ +│ └──────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Data Access Layer │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ MyspaceVectorStoreServiceImpl │ │ +│ │ - vector upsert/search with pgvector │ │ +│ │ - HNSW index management │ │ +│ │ - prepared statements & connection pooling │ │ +│ └──────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Database Layer │ +│ PostgreSQL with pgvector extension │ +│ - myspace_vector_items table │ +│ - HNSW indexes on embeddings │ +└─────────────────────────────────────────────────────────────┘ +``` + +## 2. Embedding Provider System + +### Provider Interface + +```java +public interface EmbeddingProvider { + float[] embed(String text) throws EmbeddingException; + List embedBatch(List texts) throws EmbeddingException; + int getDimension(); + String getProviderName(); + void validateConfiguration() throws ConfigurationException; +} +``` + +### Provider Implementations + +**OpenAIProvider** +- Uses OpenAI API (text-embedding-3-small, text-embedding-3-large) +- Batch API support (up to 2048 texts per request) +- Rate limiting: 3,000 requests per minute +- Retry logic with exponential backoff + +**HuggingFaceProvider** +- Uses Hugging Face Inference API +- Supports various models (all-MiniLM-L6-v2, all-mpnet-base-v2) +- Batch processing support +- Rate limiting based on API tier + +**LocalModelProvider** +- Uses ONNX Runtime or similar for local inference +- Models: sentence-transformers, all-MiniLM-L6-v2 +- No API calls, in-memory processing +- Configurable batch size + +### Provider Manager + +```java +public class EmbeddingProviderManager { + private final EmbeddingProvider provider; + private final EmbeddingCache cache; + private final MetricsCollector metrics; + + public float[] embed(String text) { + // Check cache first + // Call provider with retry logic + // Update metrics + // Cache result + } + + public List embedBatch(List texts) { + // Split into provider batch size + // Process with caching + // Handle partial failures + } +} +``` + +## 3. Search Engine Improvements + +### Relevance Ranking + +**Primary Ranking**: Cosine similarity (0-1 normalized) + +**Secondary Ranking (Reranking)**: +- Cross-encoder model for semantic similarity refinement +- Applied to top-k results (configurable, default 10) +- Improves ranking accuracy by 15-25% + +### Hybrid Search + +```java +public class HybridSearchEngine { + public List search( + String query, + SearchFilters filters, + HybridSearchConfig config + ) { + // 1. Vector similarity search + List vectorResults = vectorSearch(query); + + // 2. Keyword search on title/subject + List keywordResults = keywordSearch(query); + + // 3. Combine and weight results + // weight = (vectorScore * config.vectorWeight) + + // (keywordScore * config.keywordWeight) + + // 4. Apply metadata filters + // 5. Apply reranking if enabled + // 6. Return top-k results + } +} +``` + +### Metadata Filtering + +Supported filters: +- `subject`: Exact match or prefix match +- `source`: Exact match +- `date_label`: Date range filtering +- Custom JSON metadata queries + +## 4. Performance Optimization + +### Query Optimization + +1. **Prepared Statements**: Reduce query compilation overhead +2. **Connection Pooling**: HikariCP with configurable pool size +3. **Column Selection**: Only fetch necessary columns +4. **Index Strategy**: HNSW with ef_construction=200, ef_search=100 + +### Caching Strategy + +**Embedding Cache**: +- LRU eviction policy +- Configurable TTL (default 1 hour) +- Configurable max size (default 10,000 entries) +- Thread-safe using ConcurrentHashMap + +**Query Result Cache** (optional): +- Cache frequent search queries +- Invalidate on upsert operations + +### Performance Targets + +- Vector search: < 200ms (p95) for typical queries +- Embedding generation: < 500ms (p95) for single text +- Batch embedding: < 2s for 32 texts +- Cache hit rate: > 50% for typical workloads + +## 5. Configuration Management + +### Configuration Properties + +```yaml +sentri: + vector: + enabled: true + provider: openai # openai, huggingface, local + + # Provider-specific config + openai: + api-key: ${OPENAI_API_KEY} + model: text-embedding-3-small + dimension: 1536 + + huggingface: + api-key: ${HF_API_KEY} + model: sentence-transformers/all-MiniLM-L6-v2 + dimension: 384 + + local: + model-path: /models/all-MiniLM-L6-v2 + dimension: 384 + + # Common config + distance-metric: cosine # cosine, l2, inner_product + max-search-limit: 20 + + # Cache config + cache: + enabled: true + max-size: 10000 + ttl-minutes: 60 + + # Index config + hnsw: + ef-construction: 200 + ef-search: 100 + + # Reranking config + reranking: + enabled: false + model: cross-encoder/ms-marco-MiniLM-L-12-v2 + top-k: 10 + + # Hybrid search config + hybrid-search: + enabled: false + vector-weight: 0.7 + keyword-weight: 0.3 +``` + +## 6. API Design + +### Search Endpoint + +**Request**: +```json +{ + "query": "machine learning algorithms", + "limit": 10, + "similarity_threshold": 0.5, + "filters": { + "subject": "AI", + "source": "research", + "date_range": { + "start": "2024-01-01", + "end": "2024-12-31" + } + }, + "hybrid_search": { + "enabled": true, + "vector_weight": 0.7, + "keyword_weight": 0.3 + } +} +``` + +**Response**: +```json +{ + "results": [ + { + "item_id": "item-123", + "title": "Deep Learning Fundamentals", + "subject": "AI", + "source": "research", + "date_label": "2024-06-15", + "similarity": 0.92, + "embedding_model": "text-embedding-3-small", + "metadata": {...} + } + ], + "total": 1, + "query_time_ms": 145, + "cache_hit": false +} +``` + +### Upsert Endpoint + +**Request**: +```json +{ + "items": [ + { + "item_id": "item-123", + "title": "Deep Learning", + "subject": "AI", + "source": "research", + "date_label": "2024-06-15", + "text": "Machine learning is...", + "metadata": {...} + } + ], + "batch_size": 32 +} +``` + +**Response**: +```json +{ + "upserted": 1, + "failed": 0, + "errors": [], + "time_ms": 250 +} +``` + +## 7. Database Schema + +### Updated myspace_vector_items Table + +```sql +CREATE TABLE myspace_vector_items ( + id BIGSERIAL PRIMARY KEY, + item_id VARCHAR(128) NOT NULL UNIQUE, + title VARCHAR(255) NOT NULL, + subject VARCHAR(255), + source VARCHAR(255) NOT NULL, + date_label VARCHAR(128) NOT NULL, + embedding_model VARCHAR(255) NOT NULL, + embedding VECTOR(1536) NOT NULL, + metadata_json TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- HNSW index for fast similarity search +CREATE INDEX idx_myspace_vector_items_embedding +ON myspace_vector_items USING hnsw (embedding vector_cosine_ops) +WITH (ef_construction = 200, ef_search = 100); + +-- Indexes for filtering +CREATE INDEX idx_myspace_vector_items_subject +ON myspace_vector_items(subject); + +CREATE INDEX idx_myspace_vector_items_source +ON myspace_vector_items(source); + +CREATE INDEX idx_myspace_vector_items_date_label +ON myspace_vector_items(date_label); + +CREATE INDEX idx_myspace_vector_items_embedding_model +ON myspace_vector_items(embedding_model); +``` + +## 8. Error Handling + +### Retry Strategy + +```java +public class RetryPolicy { + private static final int MAX_RETRIES = 3; + private static final long INITIAL_BACKOFF_MS = 100; + + public static T executeWithRetry( + Callable operation, + String operationName + ) throws Exception { + long backoff = INITIAL_BACKOFF_MS; + for (int attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + return operation.call(); + } catch (RateLimitException e) { + if (attempt < MAX_RETRIES - 1) { + Thread.sleep(backoff); + backoff *= 2; + } else { + throw e; + } + } + } + } +} +``` + +### Error Types + +- `EmbeddingException`: Embedding generation failed +- `VectorStoreException`: Database operation failed +- `ConfigurationException`: Invalid configuration +- `RateLimitException`: Provider rate limit exceeded +- `ValidationException`: Input validation failed + +## 9. Monitoring & Metrics + +### Metrics to Collect + +1. **Query Metrics**: + - Query latency (min, max, avg, p95, p99) + - Query count by type (vector, keyword, hybrid) + - Result count distribution + +2. **Embedding Metrics**: + - API call count by provider + - Embedding latency by provider + - Error rate by provider + - Batch size distribution + +3. **Cache Metrics**: + - Cache hit rate + - Cache miss rate + - Eviction count + - Cache size + +4. **Index Metrics**: + - Index size + - Queries per second + - Index efficiency + +### Prometheus Metrics + +``` +# Query metrics +vector_search_latency_ms{quantile="0.95"} +vector_search_total +vector_search_errors_total + +# Embedding metrics +embedding_api_calls_total{provider="openai"} +embedding_latency_ms{provider="openai"} +embedding_errors_total{provider="openai"} + +# Cache metrics +embedding_cache_hits_total +embedding_cache_misses_total +embedding_cache_size_bytes +embedding_cache_evictions_total +``` + +## 10. Implementation Phases (12+ Commits) + +### Phase 1: Foundation (Commits 1-3) +1. **Commit 1**: Create embedding provider interfaces and base classes +2. **Commit 2**: Implement OpenAI provider with batch support +3. **Commit 3**: Implement HuggingFace provider + +### Phase 2: Caching & Local Models (Commits 4-5) +4. **Commit 4**: Implement embedding cache with LRU eviction +5. **Commit 5**: Implement local model provider + +### Phase 3: Search Improvements (Commits 6-8) +6. **Commit 6**: Add relevance ranking and similarity scoring +7. **Commit 7**: Implement hybrid search (vector + keyword) +8. **Commit 8**: Add metadata filtering and query optimization + +### Phase 4: Configuration & Management (Commits 9-10) +9. **Commit 9**: Create configuration management system +10. **Commit 10**: Implement provider manager with routing + +### Phase 5: Monitoring & Resilience (Commits 11-12) +11. **Commit 11**: Add metrics collection and Prometheus exposure +12. **Commit 12**: Implement error handling, retry logic, and health checks + +### Phase 6: API & Documentation (Commits 13+) +13. **Commit 13**: Update API endpoints with new request/response contracts +14. **Commit 14**: Add comprehensive documentation and examples +15. **Commit 15**: Add integration tests and performance benchmarks + +## 11. Testing Strategy + +### Unit Tests +- Provider implementations +- Cache eviction logic +- Configuration validation +- Error handling + +### Integration Tests +- End-to-end search flow +- Batch operations +- Provider failover +- Database operations + +### Performance Tests +- Query latency benchmarks +- Cache hit rate validation +- Batch embedding efficiency +- Index performance + +## 12. Migration Path + +1. Deploy new code with feature flags disabled +2. Enable embedding cache first (low risk) +3. Enable new providers (with fallback to existing) +4. Enable hybrid search (optional feature) +5. Enable reranking (optional feature) +6. Monitor metrics and adjust configuration + diff --git a/.kiro/specs/ml-vector-improvements/requirements.md b/.kiro/specs/ml-vector-improvements/requirements.md new file mode 100644 index 0000000..4106639 --- /dev/null +++ b/.kiro/specs/ml-vector-improvements/requirements.md @@ -0,0 +1,256 @@ +# ML Vector Search Improvements - Requirements Document + +## Introduction + +The Sentri backend currently uses PostgreSQL with pgvector extension for vector storage and similarity search. This feature aims to enhance the vector search system by improving search accuracy and relevance ranking, supporting multiple embedding model providers, optimizing query performance, and fixing existing bugs in the vector store implementation. + +The improvements will enable the system to: +- Support multiple embedding model providers (OpenAI, Hugging Face, local models) +- Improve search relevance through better ranking algorithms +- Optimize query performance and reduce memory usage +- Fix bugs in the current vector store implementation +- Provide flexible configuration for different use cases + +## Glossary + +- **Vector_Store**: The PostgreSQL database with pgvector extension that stores embeddings and metadata +- **Embedding_Model**: A machine learning model that converts text into vector representations +- **Embedding_Provider**: A service that generates embeddings (e.g., OpenAI, Hugging Face, local) +- **Similarity_Score**: A numerical value (0-1) indicating how similar two vectors are +- **Relevance_Ranking**: The process of ordering search results by relevance to the query +- **Query_Vector**: The embedding representation of a search query +- **Vector_Dimension**: The size of the embedding vector (e.g., 384, 1536) +- **Distance_Metric**: The mathematical method for calculating similarity (cosine, L2, inner product) +- **Metadata**: Additional information stored with each vector item (title, subject, source, date) +- **HNSW_Index**: Hierarchical Navigable Small World index for fast approximate nearest neighbor search +- **Embedding_Cache**: In-memory cache of recently generated embeddings to reduce API calls +- **Reranking**: A secondary ranking step that refines initial search results for better accuracy +- **Hybrid_Search**: Combining vector similarity search with keyword-based filtering +- **Batch_Embedding**: Processing multiple texts into embeddings in a single operation +- **Model_Configuration**: Settings that define how an embedding model operates (dimension, provider, API key) + +## Requirements + +### Requirement 1: Support Multiple Embedding Model Providers + +**User Story:** As a developer, I want to use different embedding model providers, so that I can choose the best model for my use case and avoid vendor lock-in. + +#### Acceptance Criteria + +1. WHEN an embedding model provider is configured, THE Embedding_Provider_Manager SHALL support OpenAI, Hugging Face, and local model providers +2. WHEN a request is made to generate embeddings, THE Embedding_Provider_Manager SHALL route to the configured provider +3. WHEN switching between providers, THE Vector_Store SHALL support embeddings of different dimensions +4. WHERE a local embedding model is configured, THE Embedding_Provider_Manager SHALL load and cache the model in memory +5. WHEN an embedding provider API call fails, THE Embedding_Provider_Manager SHALL return a descriptive error with retry information +6. THE Embedding_Provider_Manager SHALL validate that the embedding dimension matches the configured Vector_Store dimension +7. WHEN multiple embedding models are used, THE Vector_Store SHALL track which model generated each embedding in the embedding_model field + +#### Acceptance Criteria (Continued) + +8. WHERE batch embedding is supported by the provider, THE Embedding_Provider_Manager SHALL process multiple texts in a single API call +9. WHEN an embedding provider is configured, THE system configuration SHALL allow specifying provider type, API credentials, model name, and vector dimension +10. THE Embedding_Provider_Manager SHALL implement provider-specific error handling and rate limiting + +### Requirement 2: Improve Search Accuracy and Relevance Ranking + +**User Story:** As a user, I want search results to be more relevant and accurate, so that I can find the information I need more quickly. + +#### Acceptance Criteria + +1. WHEN a vector search is performed, THE Search_Engine SHALL return results ranked by similarity score in descending order +2. WHEN search results are returned, THE Search_Engine SHALL include the similarity score (0-1) for each result +3. WHEN a search query is provided, THE Search_Engine SHALL support filtering results by metadata fields (subject, source, date_label) +4. WHERE reranking is enabled, THE Search_Engine SHALL apply a secondary ranking algorithm to refine top-k results +5. WHEN reranking is applied, THE Search_Engine SHALL use a cross-encoder model or semantic similarity metric to improve ranking accuracy +6. WHEN search results are returned, THE Search_Engine SHALL support configurable similarity thresholds to filter low-relevance results +7. WHEN a search is performed with metadata filters, THE Search_Engine SHALL combine vector similarity with metadata matching +8. THE Search_Engine SHALL support hybrid search combining vector similarity with keyword-based matching on title and subject fields +9. WHEN hybrid search is used, THE Search_Engine SHALL weight vector similarity and keyword relevance according to configurable parameters +10. WHEN search results are limited, THE Search_Engine SHALL enforce a maximum result limit (configurable, default 20) + +### Requirement 3: Optimize Query Performance + +**User Story:** As an operator, I want vector search queries to execute quickly, so that users experience responsive search results. + +#### Acceptance Criteria + +1. WHEN a vector search query is executed, THE Query_Optimizer SHALL complete within 200ms for typical queries (p95) +2. WHEN the Vector_Store is initialized, THE Query_Optimizer SHALL create and maintain HNSW indexes on embedding vectors +3. WHEN multiple queries are executed, THE Query_Optimizer SHALL cache frequently accessed embeddings in memory +4. WHERE batch operations are performed, THE Query_Optimizer SHALL process multiple upserts in a single transaction +5. WHEN the Vector_Store is queried, THE Query_Optimizer SHALL use prepared statements to reduce query compilation overhead +6. WHEN search results are retrieved, THE Query_Optimizer SHALL only fetch necessary columns (item_id, title, subject, source, date_label, similarity) +7. WHEN the database connection pool is configured, THE Query_Optimizer SHALL use connection pooling to reduce connection overhead +8. WHEN vector indexes are created, THE Query_Optimizer SHALL use appropriate index parameters (ef_construction, ef_search) for HNSW +9. WHEN the Vector_Store grows large, THE Query_Optimizer SHALL support partitioning or archiving of old vectors +10. THE Query_Optimizer SHALL provide metrics on query execution time, cache hit rate, and index efficiency + +### Requirement 4: Fix Vector Store Implementation Bugs + +**User Story:** As a developer, I want the vector store to be reliable and correct, so that I can trust the search results and data integrity. + +#### Acceptance Criteria + +1. WHEN a vector is upserted, THE Vector_Store SHALL validate that the embedding dimension matches the configured dimension +2. WHEN a vector is upserted, THE Vector_Store SHALL validate that the embedding is not null or empty +3. WHEN a vector is upserted with duplicate item_id, THE Vector_Store SHALL update the existing record (upsert semantics) +4. WHEN a vector search is performed, THE Vector_Store SHALL return results sorted by similarity in descending order +5. WHEN the Vector_Store is initialized, THE Vector_Store SHALL create the myspace_vector_items table if it does not exist +6. WHEN the Vector_Store is initialized, THE Vector_Store SHALL create HNSW indexes if they do not exist +7. IF a database connection fails, THEN THE Vector_Store SHALL handle the error gracefully and provide retry logic +8. WHEN metadata_json is stored, THE Vector_Store SHALL validate that it is valid JSON or null +9. WHEN vectors are searched, THE Vector_Store SHALL handle edge cases (empty result set, single result, max limit exceeded) +10. WHEN the Vector_Store is queried, THE Vector_Store SHALL use consistent distance metrics (cosine, L2, inner product) + +### Requirement 5: Enhance Vector Store Configuration + +**User Story:** As an operator, I want to configure the vector store flexibly, so that I can optimize for different use cases and environments. + +#### Acceptance Criteria + +1. WHEN the application starts, THE Configuration_Manager SHALL load vector store settings from application properties +2. THE Configuration_Manager SHALL support configuring embedding provider type (openai, huggingface, local) +3. THE Configuration_Manager SHALL support configuring embedding model name and API credentials +4. THE Configuration_Manager SHALL support configuring vector dimension (default 384) +5. THE Configuration_Manager SHALL support configuring distance metric (cosine, l2, inner_product) +6. THE Configuration_Manager SHALL support configuring maximum search result limit (default 20) +7. THE Configuration_Manager SHALL support enabling/disabling vector store functionality +8. THE Configuration_Manager SHALL support configuring HNSW index parameters (ef_construction, ef_search) +9. THE Configuration_Manager SHALL support configuring embedding cache size and TTL +10. WHERE configuration is invalid, THE Configuration_Manager SHALL fail fast with descriptive error messages + +### Requirement 6: Add Embedding Caching + +**User Story:** As an operator, I want to reduce API calls to embedding providers, so that I can lower costs and improve performance. + +#### Acceptance Criteria + +1. WHEN an embedding is requested, THE Embedding_Cache SHALL check if the embedding exists in cache before calling the provider +2. WHEN an embedding is generated, THE Embedding_Cache SHALL store it in memory with a configurable TTL +3. WHEN the cache reaches maximum size, THE Embedding_Cache SHALL evict least-recently-used entries +4. WHEN cache statistics are requested, THE Embedding_Cache SHALL report hit rate, miss rate, and size +5. WHERE cache is enabled, THE Embedding_Cache SHALL reduce API calls to embedding providers by at least 50% for typical workloads +6. WHEN the application restarts, THE Embedding_Cache SHALL be cleared (no persistence required) +7. WHEN an embedding is cached, THE Embedding_Cache SHALL use a hash of the text and model name as the cache key +8. THE Embedding_Cache SHALL be thread-safe for concurrent access + +### Requirement 7: Support Batch Embedding Operations + +**User Story:** As a developer, I want to generate embeddings for multiple texts efficiently, so that I can reduce latency and API costs. + +#### Acceptance Criteria + +1. WHEN a batch embedding request is made, THE Batch_Embedder SHALL accept a list of texts and generate embeddings for all of them +2. WHEN batch embedding is performed, THE Batch_Embedder SHALL use the embedding provider's batch API if available +3. WHEN batch embedding is performed, THE Batch_Embedder SHALL return embeddings in the same order as the input texts +4. WHEN a batch embedding request exceeds the provider's batch size limit, THE Batch_Embedder SHALL split the request into multiple batches +5. WHEN batch embedding is performed, THE Batch_Embedder SHALL cache results to avoid duplicate API calls +6. WHEN batch embedding fails for some items, THE Batch_Embedder SHALL return partial results with error information for failed items +7. WHEN batch embedding is performed, THE Batch_Embedder SHALL support configurable batch size (default 32) +8. THE Batch_Embedder SHALL be more efficient than sequential embedding requests (at least 2x faster for typical workloads) + +### Requirement 8: Improve Vector Search API Contract + +**User Story:** As a developer, I want a clear and flexible API for vector search, so that I can easily integrate vector search into my application. + +#### Acceptance Criteria + +1. WHEN a vector search request is made, THE Search_API SHALL accept a query string or embedding vector +2. WHERE a query string is provided, THE Search_API SHALL generate an embedding using the configured provider +3. WHEN a search request is made, THE Search_API SHALL support optional metadata filters (subject, source, date_label) +4. WHEN a search request is made, THE Search_API SHALL support configurable result limit and similarity threshold +5. WHEN search results are returned, THE Search_API SHALL include item_id, title, subject, source, date_label, similarity, and metadata +6. WHEN a vector upsert request is made, THE Search_API SHALL accept a list of items with embeddings or text content +7. WHERE text content is provided instead of embeddings, THE Search_API SHALL generate embeddings automatically +8. WHEN a vector upsert request is made, THE Search_API SHALL support batch operations with configurable batch size +9. WHEN a vector upsert response is returned, THE Search_API SHALL include the number of items upserted and success/failure status +10. THE Search_API SHALL validate all input parameters and return descriptive error messages for invalid requests + +### Requirement 9: Add Vector Store Monitoring and Metrics + +**User Story:** As an operator, I want to monitor vector store performance, so that I can identify and fix performance issues. + +#### Acceptance Criteria + +1. WHEN vector store operations are performed, THE Metrics_Collector SHALL track query execution time (min, max, avg, p95, p99) +2. WHEN vector store operations are performed, THE Metrics_Collector SHALL track the number of queries, upserts, and errors +3. WHEN embedding generation is performed, THE Metrics_Collector SHALL track API call count, latency, and error rate by provider +4. WHEN the embedding cache is used, THE Metrics_Collector SHALL track cache hit rate, miss rate, and eviction count +5. WHEN vector search is performed, THE Metrics_Collector SHALL track result count, similarity score distribution, and filter effectiveness +6. WHEN the Vector_Store is queried, THE Metrics_Collector SHALL track index efficiency (queries per second, index size) +7. THE Metrics_Collector SHALL expose metrics in a format compatible with Prometheus or similar monitoring systems +8. WHEN metrics are queried, THE Metrics_Collector SHALL provide aggregated statistics over configurable time windows + +### Requirement 10: Support Multiple Distance Metrics + +**User Story:** As a developer, I want to choose the distance metric that best fits my use case, so that I can optimize search accuracy. + +#### Acceptance Criteria + +1. WHEN the Vector_Store is configured, THE Distance_Metric_Manager SHALL support cosine, L2, and inner product distance metrics +2. WHEN a vector search is performed, THE Distance_Metric_Manager SHALL use the configured distance metric consistently +3. WHEN the distance metric is changed, THE Distance_Metric_Manager SHALL validate that existing indexes are compatible +4. WHEN similarity scores are calculated, THE Distance_Metric_Manager SHALL normalize scores to a 0-1 range for consistency +5. WHEN search results are returned, THE Distance_Metric_Manager SHALL include the distance metric used in the response metadata +6. THE Distance_Metric_Manager SHALL document the mathematical definition and use cases for each supported metric + +### Requirement 11: Add Vector Store Health Checks + +**User Story:** As an operator, I want to verify that the vector store is healthy and operational, so that I can detect issues early. + +#### Acceptance Criteria + +1. WHEN a health check is performed, THE Health_Checker SHALL verify that the database connection is active +2. WHEN a health check is performed, THE Health_Checker SHALL verify that the pgvector extension is installed +3. WHEN a health check is performed, THE Health_Checker SHALL verify that the myspace_vector_items table exists +4. WHEN a health check is performed, THE Health_Checker SHALL verify that HNSW indexes are present and valid +5. WHEN a health check is performed, THE Health_Checker SHALL verify that the embedding provider is accessible +6. WHEN a health check is performed, THE Health_Checker SHALL measure query latency and report if it exceeds thresholds +7. WHEN a health check fails, THE Health_Checker SHALL provide detailed error information for troubleshooting +8. THE Health_Checker SHALL be callable via a REST endpoint for integration with monitoring systems + +### Requirement 12: Support Vector Store Maintenance Operations + +**User Story:** As an operator, I want to perform maintenance operations on the vector store, so that I can keep it healthy and efficient. + +#### Acceptance Criteria + +1. WHEN a maintenance operation is requested, THE Maintenance_Manager SHALL support rebuilding HNSW indexes +2. WHEN a maintenance operation is requested, THE Maintenance_Manager SHALL support vacuuming the database to reclaim space +3. WHEN a maintenance operation is requested, THE Maintenance_Manager SHALL support analyzing table statistics for query optimization +4. WHEN a maintenance operation is requested, THE Maintenance_Manager SHALL support archiving or deleting old vectors based on date +5. WHEN a maintenance operation is requested, THE Maintenance_Manager SHALL support exporting vectors for backup or analysis +6. WHEN a maintenance operation is performed, THE Maintenance_Manager SHALL provide progress updates and completion status +7. WHEN a maintenance operation is performed, THE Maintenance_Manager SHALL not block normal vector store operations (non-blocking) +8. THE Maintenance_Manager SHALL be callable via REST endpoints or scheduled jobs + +### Requirement 13: Improve Error Handling and Resilience + +**User Story:** As a developer, I want robust error handling and resilience, so that the system can recover from failures gracefully. + +#### Acceptance Criteria + +1. WHEN an embedding provider API call fails, THE Error_Handler SHALL implement exponential backoff retry logic +2. WHEN a database connection fails, THE Error_Handler SHALL implement connection retry logic with configurable attempts +3. WHEN a vector search fails, THE Error_Handler SHALL return a descriptive error message with the root cause +4. WHEN an embedding generation fails, THE Error_Handler SHALL fall back to a default embedding or return an error +5. WHEN a batch operation partially fails, THE Error_Handler SHALL return partial results with error information for failed items +6. WHEN rate limiting is encountered, THE Error_Handler SHALL implement adaptive rate limiting to respect provider limits +7. WHEN a timeout occurs, THE Error_Handler SHALL cancel the operation and return a timeout error +8. THE Error_Handler SHALL log all errors with sufficient context for debugging + +### Requirement 14: Add Vector Store Documentation and Examples + +**User Story:** As a developer, I want clear documentation and examples, so that I can easily integrate vector search into my application. + +#### Acceptance Criteria + +1. THE Documentation SHALL include setup instructions for each embedding provider (OpenAI, Hugging Face, local) +2. THE Documentation SHALL include API documentation for all vector search endpoints +3. THE Documentation SHALL include configuration examples for different use cases +4. THE Documentation SHALL include performance tuning guidelines +5. THE Documentation SHALL include troubleshooting guides for common issues +6. THE Documentation SHALL include code examples for common operations (search, upsert, batch operations) +7. THE Documentation SHALL include migration guides for upgrading from the current implementation +8. THE Documentation SHALL be maintained and updated with each release + diff --git a/.kiro/specs/ml-vector-improvements/tasks.md b/.kiro/specs/ml-vector-improvements/tasks.md new file mode 100644 index 0000000..bd44b13 --- /dev/null +++ b/.kiro/specs/ml-vector-improvements/tasks.md @@ -0,0 +1,323 @@ +# Implementation Plan: ML Vector Search Improvements + +## Overview + +This implementation plan breaks down the ML Vector Search Improvements feature into 6 phases with 15+ tasks. The feature enhances the vector search system by supporting multiple embedding providers, improving search accuracy, optimizing performance, and adding comprehensive monitoring. Implementation follows a bottom-up approach: foundation → caching → search improvements → configuration → monitoring → API updates. + +## Phase 1: Foundation (Commits 1-3) + +### Objective +Establish the embedding provider abstraction layer and implement core providers (OpenAI, HuggingFace). + +--- + +- [ ] 1. Create embedding provider interfaces and base classes + - Create `EmbeddingProvider` interface with methods: `embed()`, `embedBatch()`, `getDimension()`, `getProviderName()`, `validateConfiguration()` + - Create `EmbeddingException` and `ConfigurationException` custom exceptions + - Create abstract base class `BaseEmbeddingProvider` with common retry logic and error handling + - Create `EmbeddingProviderFactory` for provider instantiation + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProvider.java`, `BaseEmbeddingProvider.java`, `EmbeddingProviderFactory.java`, exception classes + - _Requirements: 1.1, 1.5, 1.10_ + +- [ ] 2. Implement OpenAI embedding provider with batch support + - Implement `OpenAIProvider` class using OpenAI API (text-embedding-3-small, text-embedding-3-large) + - Add batch API support (up to 2048 texts per request) + - Implement rate limiting (3,000 requests per minute) + - Add exponential backoff retry logic for transient failures + - Implement dimension validation (1536 for text-embedding-3-small, 3072 for text-embedding-3-large) + - Add configuration validation for API key and model name + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/provider/OpenAIProvider.java` + - Files to modify: `pom.xml` (add OpenAI client library dependency) + - _Requirements: 1.1, 1.2, 1.8, 1.10_ + +- [ ] 3. Implement HuggingFace embedding provider + - Implement `HuggingFaceProvider` class using Hugging Face Inference API + - Support multiple models (all-MiniLM-L6-v2, all-mpnet-base-v2, etc.) + - Add batch processing support + - Implement rate limiting based on API tier + - Add configuration validation for API key and model name + - Implement dimension detection from model metadata + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/provider/HuggingFaceProvider.java` + - Files to modify: `pom.xml` (add HuggingFace client library dependency) + - _Requirements: 1.1, 1.8, 1.10_ + +--- + +## Phase 2: Caching & Local Models (Commits 4-5) + +### Objective +Implement embedding cache with LRU eviction and local model provider for offline embedding generation. + +--- + +- [ ] 4. Implement embedding cache with LRU eviction + - Create `EmbeddingCache` class using `ConcurrentHashMap` for thread-safe access + - Implement LRU eviction policy using `LinkedHashMap` or similar + - Add configurable TTL (default 1 hour) with expiration checking + - Add configurable max size (default 10,000 entries) + - Implement cache key generation: `hash(text + model_name)` + - Add cache statistics tracking: hit count, miss count, eviction count, current size + - Implement cache invalidation on upsert operations + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/cache/EmbeddingCache.java`, `CacheEntry.java`, `CacheStatistics.java` + - _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7, 6.8_ + +- [ ] 5. Implement local model embedding provider + - Implement `LocalModelProvider` class using ONNX Runtime or similar + - Support sentence-transformers models (all-MiniLM-L6-v2, etc.) + - Add model loading and caching in memory + - Implement batch processing with configurable batch size + - Add model path configuration and validation + - Implement dimension detection from model metadata + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/provider/LocalModelProvider.java` + - Files to modify: `pom.xml` (add ONNX Runtime or similar dependency) + - _Requirements: 1.1, 1.4, 1.10_ + +--- + +## Phase 3: Search Improvements (Commits 6-8) + +### Objective +Enhance search accuracy through relevance ranking, hybrid search, and metadata filtering. + +--- + +- [ ] 6. Add relevance ranking and similarity scoring + - Create `SimilarityScorer` class for cosine similarity calculation + - Implement similarity score normalization to 0-1 range + - Create `RelevanceRanker` class for primary ranking by cosine similarity + - Add support for configurable similarity threshold filtering + - Implement cross-encoder model support for reranking (optional, configurable) + - Add reranking logic to refine top-k results (default k=10) + - Create `SearchResult` DTO with similarity score and ranking metadata + - Files to create: `backend/src/main/java/com/sentri/backend/search/SimilarityScorer.java`, `RelevanceRanker.java`, `SearchResult.java` + - Files to modify: `backend/src/main/java/com/sentri/backend/dto/response/MyspaceVectorSearchResponse.java` (add similarity field) + - _Requirements: 2.1, 2.2, 2.6_ + +- [ ] 7. Implement hybrid search (vector + keyword) + - Create `HybridSearchEngine` class combining vector and keyword search + - Implement vector similarity search using existing pgvector queries + - Implement keyword search on title and subject fields + - Add configurable weighting: `score = (vectorScore * vectorWeight) + (keywordScore * keywordWeight)` + - Implement result deduplication and merging + - Add support for hybrid search configuration (enabled, weights) + - Create `HybridSearchConfig` configuration class + - Files to create: `backend/src/main/java/com/sentri/backend/search/HybridSearchEngine.java`, `HybridSearchConfig.java` + - Files to modify: `backend/src/main/java/com/sentri/backend/dto/request/MyspaceVectorSearchRequest.java` (add hybrid search fields) + - _Requirements: 2.8, 2.9_ + +- [ ] 8. Add metadata filtering and query optimization + - Create `MetadataFilter` class supporting subject, source, date_label, and custom JSON metadata queries + - Implement filter validation and SQL generation + - Add support for date range filtering + - Optimize queries to use prepared statements and column selection + - Implement connection pooling configuration (HikariCP) + - Add HNSW index parameter configuration (ef_construction=200, ef_search=100) + - Create `QueryOptimizer` class for query planning and optimization + - Files to create: `backend/src/main/java/com/sentri/backend/search/MetadataFilter.java`, `QueryOptimizer.java` + - Files to modify: `backend/src/main/java/com/sentri/backend/dto/request/MyspaceVectorSearchRequest.java` (add filter fields) + - _Requirements: 2.3, 3.1, 3.5, 3.6, 3.7, 3.8_ + +--- + +## Phase 4: Configuration & Management (Commits 9-10) + +### Objective +Create configuration management system and implement provider manager with routing. + +--- + +- [ ] 9. Create configuration management system + - Create `VectorStoreProperties` configuration class with all vector store settings + - Add provider configuration: type (openai, huggingface, local), credentials, model name, dimension + - Add cache configuration: enabled, max-size, ttl-minutes + - Add index configuration: ef-construction, ef-search + - Add reranking configuration: enabled, model, top-k + - Add hybrid search configuration: enabled, vector-weight, keyword-weight + - Add distance metric configuration: cosine, l2, inner_product + - Implement configuration validation with descriptive error messages + - Add configuration loading from `application.yml` with environment variable support + - Files to create: `backend/src/main/java/com/sentri/backend/config/VectorStoreProperties.java`, nested configuration classes + - Files to modify: `backend/src/main/resources/application.yml` (add vector store configuration section) + - _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 5.10_ + +- [ ] 10. Implement provider manager with routing + - Create `EmbeddingProviderManager` class that routes to configured provider + - Implement `embed()` method with cache checking, provider call, and result caching + - Implement `embedBatch()` method with batch splitting, caching, and partial failure handling + - Add provider validation on startup + - Implement dimension validation against configured dimension + - Add error handling with descriptive messages + - Implement metrics collection integration (to be used in Phase 5) + - Files to create: `backend/src/main/java/com/sentri/backend/embedding/EmbeddingProviderManager.java` + - Files to modify: `backend/src/main/java/com/sentri/backend/config/VectorStoreProperties.java` (reference in manager) + - _Requirements: 1.2, 1.3, 1.6, 1.7, 1.8, 1.10_ + +--- + +## Phase 5: Monitoring & Resilience (Commits 11-12) + +### Objective +Add metrics collection, Prometheus exposure, error handling, retry logic, and health checks. + +--- + +- [ ] 11. Add metrics collection and Prometheus exposure + - Create `MetricsCollector` class for tracking all metrics + - Implement query metrics: latency (min, max, avg, p95, p99), count by type, result count distribution + - Implement embedding metrics: API call count by provider, latency by provider, error rate by provider, batch size distribution + - Implement cache metrics: hit rate, miss rate, eviction count, cache size + - Implement index metrics: queries per second, index efficiency + - Create Prometheus metrics using Micrometer: `vector_search_latency_ms`, `vector_search_total`, `embedding_api_calls_total`, `embedding_cache_hits_total`, etc. + - Add metrics endpoint for Prometheus scraping + - Integrate metrics collection into `EmbeddingProviderManager` and search operations + - Files to create: `backend/src/main/java/com/sentri/backend/metrics/MetricsCollector.java`, `PrometheusMetricsExporter.java` + - Files to modify: `pom.xml` (add Micrometer Prometheus dependency) + - _Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8_ + +- [ ] 12. Implement error handling, retry logic, and health checks + - Create `RetryPolicy` class with exponential backoff (max 3 retries, initial backoff 100ms) + - Implement `RateLimitException` and `TimeoutException` custom exceptions + - Add retry logic to `EmbeddingProviderManager` for transient failures + - Implement connection retry logic for database failures + - Add partial failure handling for batch operations + - Create `VectorStoreHealthChecker` class for health checks + - Implement health check validations: database connection, pgvector extension, table existence, index existence, provider accessibility, query latency + - Create `/health/vector-store` endpoint returning detailed health status + - Add error logging with sufficient context for debugging + - Files to create: `backend/src/main/java/com/sentri/backend/error/RetryPolicy.java`, `VectorStoreHealthChecker.java`, exception classes + - Files to modify: `backend/src/main/java/com/sentri/backend/controller/HealthController.java` (add vector store health endpoint) + - _Requirements: 4.7, 11.1, 11.2, 11.3, 11.4, 11.5, 11.6, 11.7, 11.8, 13.1, 13.2, 13.3, 13.4, 13.5, 13.6, 13.7, 13.8_ + +--- + +## Phase 6: API & Documentation (Commits 13+) + +### Objective +Update API endpoints with new request/response contracts, add documentation, and integration tests. + +--- + +- [ ] 13. Update API endpoints with new request/response contracts + - Update `MyspaceVectorSearchRequest` DTO: add filters, similarity_threshold, hybrid_search config + - Update `MyspaceVectorSearchResponse` DTO: add similarity scores, embedding_model, query_time_ms, cache_hit flag + - Update `MyspaceVectorUpsertRequest` DTO: add batch_size, support text-to-embedding conversion + - Update `MyspaceVectorUpsertResponse` DTO: add upserted count, failed count, error details, time_ms + - Update `MyspaceVectorItemRequest` DTO: add metadata_json field + - Implement request validation with descriptive error messages + - Update `MyspaceVectorController` to use new service layer with `VectorSearchService` + - Add endpoint for health checks: `GET /api/vector-store/health` + - Files to modify: `backend/src/main/java/com/sentri/backend/dto/request/MyspaceVectorSearchRequest.java`, response DTOs, `MyspaceVectorController.java` + - _Requirements: 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 8.10_ + +- [ ] 14. Create VectorSearchService orchestrating all components + - Create `VectorSearchService` class orchestrating embedding generation, caching, search, and ranking + - Implement `search()` method: generate query embedding, apply filters, perform hybrid search, apply reranking, return results + - Implement `upsert()` method: validate inputs, generate embeddings (batch), store in database, update cache + - Implement `health()` method: delegate to `VectorStoreHealthChecker` + - Add transaction management for batch operations + - Integrate metrics collection + - Integrate error handling and retry logic + - Files to create: `backend/src/main/java/com/sentri/backend/service/VectorSearchService.java` + - _Requirements: 1.2, 2.1, 3.1, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9_ + +- [ ] 15. Add comprehensive documentation and examples + - Create `VECTOR_SEARCH_SETUP.md`: setup instructions for each provider (OpenAI, HuggingFace, local) + - Create `VECTOR_SEARCH_API.md`: API documentation for all endpoints with request/response examples + - Create `VECTOR_SEARCH_CONFIG.md`: configuration examples for different use cases + - Create `VECTOR_SEARCH_TUNING.md`: performance tuning guidelines + - Create `VECTOR_SEARCH_TROUBLESHOOTING.md`: troubleshooting guides for common issues + - Create `VECTOR_SEARCH_EXAMPLES.md`: code examples for common operations (search, upsert, batch) + - Create `VECTOR_SEARCH_MIGRATION.md`: migration guide from current implementation + - Add inline code documentation (JavaDoc) for all public classes and methods + - Files to create: `backend/docs/VECTOR_SEARCH_*.md` + - _Requirements: 14.1, 14.2, 14.3, 14.4, 14.5, 14.6, 14.7, 14.8_ + +- [ ] 16. Add integration tests and performance benchmarks + - Create integration tests for end-to-end search flow with real database + - Create integration tests for batch operations + - Create integration tests for provider failover and error handling + - Create integration tests for database operations (upsert, search, filtering) + - Create performance benchmarks for query latency (target: < 200ms p95) + - Create performance benchmarks for embedding generation (target: < 500ms p95 for single, < 2s for batch of 32) + - Create performance benchmarks for cache hit rate (target: > 50%) + - Create tests for metadata filtering and hybrid search + - Files to create: `backend/src/test/java/com/sentri/backend/integration/VectorSearchIntegrationTest.java`, benchmark classes + - _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10_ + +--- + +## Checkpoint Tasks + +- [ ] 17. Checkpoint - Phase 1-2 Complete + - Ensure all embedding providers are implemented and tested + - Ensure cache is working with proper LRU eviction + - Ensure all tests pass + - Ask the user if questions arise + +- [ ] 18. Checkpoint - Phase 3-4 Complete + - Ensure search improvements are working (relevance ranking, hybrid search, filtering) + - Ensure configuration management is working + - Ensure provider manager is routing correctly + - Ensure all tests pass + - Ask the user if questions arise + +- [ ] 19. Checkpoint - Phase 5-6 Complete + - Ensure metrics are being collected and exposed + - Ensure health checks are working + - Ensure API endpoints are updated with new contracts + - Ensure documentation is complete + - Ensure integration tests pass + - Ensure performance benchmarks meet targets + - Ask the user if questions arise + +--- + +## Implementation Notes + +### Task Dependencies + +- Phase 1 (tasks 1-3) must be completed before Phase 2 +- Phase 2 (tasks 4-5) must be completed before Phase 4 +- Phase 3 (tasks 6-8) must be completed before Phase 4 +- Phase 4 (tasks 9-10) must be completed before Phase 5 +- Phase 5 (tasks 11-12) must be completed before Phase 6 +- Phase 6 (tasks 13-16) can begin after Phase 4 is complete + +### Complexity Estimates + +- Task 1: Medium (interface design, base classes) +- Task 2: High (OpenAI API integration, batch support, retry logic) +- Task 3: High (HuggingFace API integration, batch support) +- Task 4: Medium (cache implementation, LRU eviction, thread safety) +- Task 5: High (ONNX Runtime integration, model loading) +- Task 6: Medium (similarity scoring, reranking logic) +- Task 7: Medium (hybrid search implementation, result merging) +- Task 8: Medium (metadata filtering, query optimization) +- Task 9: Low (configuration management, property classes) +- Task 10: Medium (provider manager, routing logic) +- Task 11: Medium (metrics collection, Prometheus integration) +- Task 12: High (error handling, retry logic, health checks) +- Task 13: Low (DTO updates, endpoint modifications) +- Task 14: High (service orchestration, transaction management) +- Task 15: Low (documentation writing) +- Task 16: High (integration tests, performance benchmarks) + +### Key Considerations + +1. **Thread Safety**: Embedding cache and provider manager must be thread-safe for concurrent requests +2. **Error Handling**: All external API calls must have retry logic and graceful degradation +3. **Performance**: Query latency target is < 200ms p95; cache hit rate target is > 50% +4. **Configuration**: All settings must be configurable via `application.yml` with environment variable support +5. **Monitoring**: All operations must be instrumented with metrics for observability +6. **Testing**: Each phase should have corresponding unit and integration tests +7. **Documentation**: Clear setup and usage documentation for each provider + +### Migration Strategy + +1. Deploy new code with feature flags disabled +2. Enable embedding cache first (low risk) +3. Enable new providers (with fallback to existing) +4. Enable hybrid search (optional feature) +5. Enable reranking (optional feature) +6. Monitor metrics and adjust configuration + diff --git a/README.md b/README.md index 2ccead9..58cdbca 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ -# Sentri +# Sentri 🎓 Sentri is a zero-budget cross-platform student companion for Army Institute of Technology. +[![Build Status](https://img.shields.io/badge/build-passing-brightgreen)](https://github.com/SahilKumar75/sentri) +[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) +[![Platform](https://img.shields.io/badge/platform-iOS%20%7C%20Android-lightgrey)](https://expo.dev) + This repository is currently split into 3 working parts: - `app/`: Expo React Native frontend for iPhone and Android diff --git a/app/src/lib/analytics.ts b/app/src/lib/analytics.ts new file mode 100644 index 0000000..e4d1999 --- /dev/null +++ b/app/src/lib/analytics.ts @@ -0,0 +1,185 @@ +/** + * Analytics and tracking utilities for user behavior insights + */ + +export interface AnalyticsEvent { + name: string; + properties?: Record; + timestamp: Date; + userId?: string; + sessionId: string; +} + +export class Analytics { + private static instance: Analytics; + private sessionId: string; + private userId?: string; + private events: AnalyticsEvent[] = []; + private isEnabled: boolean = true; + + private constructor() { + this.sessionId = this.generateSessionId(); + } + + static getInstance(): Analytics { + if (!Analytics.instance) { + Analytics.instance = new Analytics(); + } + return Analytics.instance; + } + + /** + * Initialize analytics with user context + */ + initialize(userId?: string, options?: { enabled?: boolean }): void { + this.userId = userId; + this.isEnabled = options?.enabled ?? true; + + if (this.isEnabled) { + this.track('app_initialized', { + platform: 'mobile', + userId: userId || 'anonymous', + }); + } + } + + /** + * Track an event + */ + track(eventName: string, properties?: Record): void { + if (!this.isEnabled) return; + + const event: AnalyticsEvent = { + name: eventName, + properties: { + ...properties, + platform: 'mobile', + timestamp: new Date().toISOString(), + }, + timestamp: new Date(), + userId: this.userId, + sessionId: this.sessionId, + }; + + this.events.push(event); + + if (__DEV__) { + console.log('Analytics Event:', event); + } + + // In production, send to analytics service + this.sendToAnalyticsService(event); + } + + /** + * Track screen view + */ + trackScreen(screenName: string, properties?: Record): void { + this.track('screen_view', { + screen_name: screenName, + ...properties, + }); + } + + /** + * Track user action + */ + trackAction(action: string, category: string, properties?: Record): void { + this.track('user_action', { + action, + category, + ...properties, + }); + } + + /** + * Track error events + */ + trackError(error: Error, context?: string): void { + this.track('error_occurred', { + error_name: error.name, + error_message: error.message, + error_stack: error.stack, + context, + }); + } + + /** + * Track performance metrics + */ + trackPerformance(metric: string, value: number, unit: string = 'ms'): void { + this.track('performance_metric', { + metric, + value, + unit, + }); + } + + /** + * Set user properties + */ + setUserProperties(properties: Record): void { + if (!this.isEnabled) return; + + this.track('user_properties_updated', { + properties, + }); + } + + /** + * Enable or disable analytics + */ + setEnabled(enabled: boolean): void { + this.isEnabled = enabled; + + if (enabled) { + this.track('analytics_enabled'); + } + } + + /** + * Get current session events + */ + getSessionEvents(): AnalyticsEvent[] { + return [...this.events]; + } + + /** + * Clear session events + */ + clearSession(): void { + this.events = []; + this.sessionId = this.generateSessionId(); + } + + private generateSessionId(): string { + return `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + } + + private async sendToAnalyticsService(event: AnalyticsEvent): Promise { + try { + // In production, implement actual analytics service integration + // Examples: Firebase Analytics, Mixpanel, Amplitude, etc. + + if (__DEV__) { + // In development, just log + return; + } + + // Example implementation: + // await fetch('/api/analytics', { + // method: 'POST', + // headers: { 'Content-Type': 'application/json' }, + // body: JSON.stringify(event), + // }); + + } catch (error) { + if (__DEV__) { + console.error('Failed to send analytics event:', error); + } + } + } +} + +// Convenience export +export const analytics = Analytics.getInstance(); \ No newline at end of file diff --git a/app/src/lib/constants.ts b/app/src/lib/constants.ts new file mode 100644 index 0000000..189e91b --- /dev/null +++ b/app/src/lib/constants.ts @@ -0,0 +1,78 @@ +/** + * Application constants and configuration + */ + +export const APP_CONFIG = { + name: 'Sentri', + version: '1.0.0', + description: 'Student companion for Army Institute of Technology', +} as const; + +export const API_ENDPOINTS = { + auth: '/api/auth', + timetable: '/api/timetable', + myspace: '/api/myspace', + hangout: '/api/hangout', + health: '/api/health', +} as const; + +export const STORAGE_KEYS = { + authToken: 'sentri_auth_token', + userProfile: 'sentri_user_profile', + timetableCache: 'sentri_timetable_cache', + myspaceCache: 'sentri_myspace_cache', + appSettings: 'sentri_app_settings', +} as const; + +export const COLORS = { + primary: '#007AFF', + secondary: '#5856D6', + success: '#34C759', + warning: '#FF9500', + error: '#FF3B30', + background: '#F2F2F7', + surface: '#FFFFFF', + text: '#000000', + textSecondary: '#8E8E93', +} as const; + +export const DIMENSIONS = { + borderRadius: 8, + spacing: { + xs: 4, + sm: 8, + md: 16, + lg: 24, + xl: 32, + }, + fontSize: { + xs: 12, + sm: 14, + md: 16, + lg: 18, + xl: 24, + xxl: 32, + }, +} as const; + +export const VALIDATION_RULES = { + email: { + required: true, + pattern: /^[A-Za-z0-9+_.-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/, + }, + password: { + minLength: 8, + requireUppercase: true, + requireLowercase: true, + requireNumber: true, + }, + phone: { + pattern: /^[+]?[1-9]\d{1,14}$/, + }, +} as const; + +export const TIMEOUTS = { + api: 10000, // 10 seconds + upload: 30000, // 30 seconds + cache: 300000, // 5 minutes +} as const; \ No newline at end of file diff --git a/app/src/lib/error-handler.ts b/app/src/lib/error-handler.ts new file mode 100644 index 0000000..b0d4ec6 --- /dev/null +++ b/app/src/lib/error-handler.ts @@ -0,0 +1,116 @@ +/** + * Centralized error handling utilities for the mobile app + */ + +export interface AppError { + code: string; + message: string; + details?: any; + timestamp: Date; +} + +export class ErrorHandler { + /** + * Creates a standardized error object + */ + static createError(code: string, message: string, details?: any): AppError { + return { + code, + message, + details, + timestamp: new Date(), + }; + } + + /** + * Handles API errors and converts them to user-friendly messages + */ + static handleApiError(error: any): AppError { + if (error.response) { + const { status, data } = error.response; + + switch (status) { + case 400: + return this.createError('VALIDATION_ERROR', + data.message || 'Invalid request data', data); + case 401: + return this.createError('UNAUTHORIZED', + 'Please log in to continue', data); + case 403: + return this.createError('FORBIDDEN', + 'You do not have permission to perform this action', data); + case 404: + return this.createError('NOT_FOUND', + 'The requested resource was not found', data); + case 500: + return this.createError('SERVER_ERROR', + 'Something went wrong on our end. Please try again later', data); + default: + return this.createError('API_ERROR', + data.message || 'An unexpected error occurred', data); + } + } + + if (error.request) { + return this.createError('NETWORK_ERROR', + 'Unable to connect to the server. Please check your internet connection'); + } + + return this.createError('UNKNOWN_ERROR', + error.message || 'An unexpected error occurred'); + } + + /** + * Handles validation errors + */ + static handleValidationError(field: string, rule: string): AppError { + const messages: Record = { + required: `${field} is required`, + email: 'Please enter a valid email address', + password: 'Password must be at least 8 characters with uppercase, lowercase, and number', + phone: 'Please enter a valid phone number', + minLength: `${field} is too short`, + maxLength: `${field} is too long`, + }; + + return this.createError('VALIDATION_ERROR', + messages[rule] || `${field} is invalid`); + } + + /** + * Logs error for debugging purposes + */ + static logError(error: AppError | Error, context?: string): void { + const logData = { + context, + error: error instanceof Error ? { + name: error.name, + message: error.message, + stack: error.stack, + } : error, + timestamp: new Date().toISOString(), + }; + + if (__DEV__) { + console.error('App Error:', logData); + } + + // In production, you might want to send this to a logging service + // LoggingService.reportError(logData); + } + + /** + * Determines if an error should be retried + */ + static shouldRetry(error: AppError): boolean { + const retryableCodes = ['NETWORK_ERROR', 'SERVER_ERROR', 'TIMEOUT_ERROR']; + return retryableCodes.includes(error.code); + } + + /** + * Gets user-friendly message for display + */ + static getUserMessage(error: AppError): string { + return error.message; + } +} \ No newline at end of file diff --git a/app/src/lib/logger.ts b/app/src/lib/logger.ts new file mode 100644 index 0000000..62453e2 --- /dev/null +++ b/app/src/lib/logger.ts @@ -0,0 +1,108 @@ +/** + * Centralized logging utility for the mobile app + */ + +export enum LogLevel { + DEBUG = 'DEBUG', + INFO = 'INFO', + WARN = 'WARN', + ERROR = 'ERROR', +} + +export interface LogEntry { + level: LogLevel; + message: string; + timestamp: Date; + context?: string; + data?: any; +} + +export class Logger { + private static instance: Logger; + private logs: LogEntry[] = []; + private maxLogs: number = 500; + private isEnabled: boolean = true; + + private constructor() {} + + static getInstance(): Logger { + if (!Logger.instance) { + Logger.instance = new Logger(); + } + return Logger.instance; + } + + private log(level: LogLevel, message: string, context?: string, data?: any): void { + if (!this.isEnabled) return; + + const entry: LogEntry = { + level, + message, + timestamp: new Date(), + context, + data, + }; + + this.logs.push(entry); + + // Keep only recent logs + if (this.logs.length > this.maxLogs) { + this.logs = this.logs.slice(-this.maxLogs); + } + + // Console output in development + if (__DEV__) { + const prefix = `[${level}]${context ? ` [${context}]` : ''}`; + const logFn = this.getConsoleFn(level); + logFn(prefix, message, data || ''); + } + } + + debug(message: string, context?: string, data?: any): void { + this.log(LogLevel.DEBUG, message, context, data); + } + + info(message: string, context?: string, data?: any): void { + this.log(LogLevel.INFO, message, context, data); + } + + warn(message: string, context?: string, data?: any): void { + this.log(LogLevel.WARN, message, context, data); + } + + error(message: string, context?: string, data?: any): void { + this.log(LogLevel.ERROR, message, context, data); + } + + getLogs(level?: LogLevel): LogEntry[] { + if (level) { + return this.logs.filter(log => log.level === level); + } + return [...this.logs]; + } + + clearLogs(): void { + this.logs = []; + } + + setEnabled(enabled: boolean): void { + this.isEnabled = enabled; + } + + private getConsoleFn(level: LogLevel): (...args: any[]) => void { + switch (level) { + case LogLevel.DEBUG: + return console.debug; + case LogLevel.INFO: + return console.info; + case LogLevel.WARN: + return console.warn; + case LogLevel.ERROR: + return console.error; + default: + return console.log; + } + } +} + +export const logger = Logger.getInstance(); \ No newline at end of file diff --git a/app/src/lib/network-monitor.ts b/app/src/lib/network-monitor.ts new file mode 100644 index 0000000..0788f28 --- /dev/null +++ b/app/src/lib/network-monitor.ts @@ -0,0 +1,165 @@ +/** + * Network connectivity monitoring utility + */ + +import NetInfo, { NetInfoState } from '@react-native-community/netinfo'; + +export enum ConnectionType { + WIFI = 'wifi', + CELLULAR = 'cellular', + NONE = 'none', + UNKNOWN = 'unknown', +} + +export interface NetworkStatus { + isConnected: boolean; + type: ConnectionType; + isInternetReachable: boolean | null; + timestamp: Date; +} + +export class NetworkMonitor { + private static instance: NetworkMonitor; + private listeners: Set<(status: NetworkStatus) => void> = new Set(); + private currentStatus: NetworkStatus | null = null; + private unsubscribe: (() => void) | null = null; + + private constructor() { + this.initialize(); + } + + static getInstance(): NetworkMonitor { + if (!NetworkMonitor.instance) { + NetworkMonitor.instance = new NetworkMonitor(); + } + return NetworkMonitor.instance; + } + + private initialize(): void { + this.unsubscribe = NetInfo.addEventListener((state: NetInfoState) => { + const status = this.parseNetInfoState(state); + this.currentStatus = status; + this.notifyListeners(status); + }); + } + + /** + * Get current network status + */ + async getStatus(): Promise { + const state = await NetInfo.fetch(); + const status = this.parseNetInfoState(state); + this.currentStatus = status; + return status; + } + + /** + * Check if device is connected to internet + */ + async isConnected(): Promise { + const status = await this.getStatus(); + return status.isConnected; + } + + /** + * Check if device is on WiFi + */ + async isWiFi(): Promise { + const status = await this.getStatus(); + return status.type === ConnectionType.WIFI; + } + + /** + * Check if device is on cellular + */ + async isCellular(): Promise { + const status = await this.getStatus(); + return status.type === ConnectionType.CELLULAR; + } + + /** + * Add listener for network status changes + */ + addListener(callback: (status: NetworkStatus) => void): () => void { + this.listeners.add(callback); + + // Immediately call with current status if available + if (this.currentStatus) { + callback(this.currentStatus); + } + + // Return unsubscribe function + return () => { + this.listeners.delete(callback); + }; + } + + /** + * Remove all listeners + */ + removeAllListeners(): void { + this.listeners.clear(); + } + + /** + * Wait for internet connection + */ + async waitForConnection(timeout: number = 30000): Promise { + return new Promise((resolve) => { + const timeoutId = setTimeout(() => { + unsubscribe(); + resolve(false); + }, timeout); + + const unsubscribe = this.addListener((status) => { + if (status.isConnected) { + clearTimeout(timeoutId); + unsubscribe(); + resolve(true); + } + }); + }); + } + + /** + * Cleanup resources + */ + destroy(): void { + if (this.unsubscribe) { + this.unsubscribe(); + this.unsubscribe = null; + } + this.removeAllListeners(); + } + + private parseNetInfoState(state: NetInfoState): NetworkStatus { + let type: ConnectionType = ConnectionType.UNKNOWN; + + if (state.type === 'wifi') { + type = ConnectionType.WIFI; + } else if (state.type === 'cellular') { + type = ConnectionType.CELLULAR; + } else if (state.type === 'none') { + type = ConnectionType.NONE; + } + + return { + isConnected: state.isConnected ?? false, + type, + isInternetReachable: state.isInternetReachable, + timestamp: new Date(), + }; + } + + private notifyListeners(status: NetworkStatus): void { + this.listeners.forEach((listener) => { + try { + listener(status); + } catch (error) { + console.error('Error in network listener:', error); + } + }); + } +} + +export const networkMonitor = NetworkMonitor.getInstance(); \ No newline at end of file diff --git a/app/src/lib/performance.ts b/app/src/lib/performance.ts new file mode 100644 index 0000000..0024429 --- /dev/null +++ b/app/src/lib/performance.ts @@ -0,0 +1,202 @@ +/** + * Performance monitoring and optimization utilities + */ + +export interface PerformanceMetric { + name: string; + value: number; + unit: string; + timestamp: Date; + metadata?: Record; +} + +export class PerformanceMonitor { + private static instance: PerformanceMonitor; + private metrics: PerformanceMetric[] = []; + private timers: Map = new Map(); + + private constructor() {} + + static getInstance(): PerformanceMonitor { + if (!PerformanceMonitor.instance) { + PerformanceMonitor.instance = new PerformanceMonitor(); + } + return PerformanceMonitor.instance; + } + + /** + * Start timing an operation + */ + startTimer(name: string): void { + this.timers.set(name, Date.now()); + } + + /** + * End timing and record metric + */ + endTimer(name: string, metadata?: Record): number { + const startTime = this.timers.get(name); + if (!startTime) { + console.warn(`Timer '${name}' was not started`); + return 0; + } + + const duration = Date.now() - startTime; + this.timers.delete(name); + + this.recordMetric(name, duration, 'ms', metadata); + return duration; + } + + /** + * Record a performance metric + */ + recordMetric(name: string, value: number, unit: string = 'ms', metadata?: Record): void { + const metric: PerformanceMetric = { + name, + value, + unit, + timestamp: new Date(), + metadata, + }; + + this.metrics.push(metric); + + // Keep only last 1000 metrics to prevent memory issues + if (this.metrics.length > 1000) { + this.metrics = this.metrics.slice(-1000); + } + + if (__DEV__) { + console.log(`Performance: ${name} = ${value}${unit}`, metadata); + } + } + + /** + * Measure function execution time + */ + async measureAsync(name: string, fn: () => Promise, metadata?: Record): Promise { + this.startTimer(name); + try { + const result = await fn(); + this.endTimer(name, { ...metadata, success: true }); + return result; + } catch (error) { + this.endTimer(name, { ...metadata, success: false, error: error.message }); + throw error; + } + } + + /** + * Measure synchronous function execution time + */ + measure(name: string, fn: () => T, metadata?: Record): T { + this.startTimer(name); + try { + const result = fn(); + this.endTimer(name, { ...metadata, success: true }); + return result; + } catch (error) { + this.endTimer(name, { ...metadata, success: false, error: error.message }); + throw error; + } + } + + /** + * Get performance statistics + */ + getStats(metricName?: string): Record { + const filteredMetrics = metricName + ? this.metrics.filter(m => m.name === metricName) + : this.metrics; + + if (filteredMetrics.length === 0) { + return { count: 0 }; + } + + const values = filteredMetrics.map(m => m.value); + const sum = values.reduce((a, b) => a + b, 0); + const avg = sum / values.length; + const min = Math.min(...values); + const max = Math.max(...values); + + // Calculate percentiles + const sorted = [...values].sort((a, b) => a - b); + const p50 = sorted[Math.floor(sorted.length * 0.5)]; + const p90 = sorted[Math.floor(sorted.length * 0.9)]; + const p95 = sorted[Math.floor(sorted.length * 0.95)]; + + return { + count: filteredMetrics.length, + sum, + avg: Math.round(avg * 100) / 100, + min, + max, + p50, + p90, + p95, + unit: filteredMetrics[0]?.unit || 'ms', + }; + } + + /** + * Get all metrics + */ + getAllMetrics(): PerformanceMetric[] { + return [...this.metrics]; + } + + /** + * Clear all metrics + */ + clear(): void { + this.metrics = []; + this.timers.clear(); + } + + /** + * Monitor React Native performance + */ + monitorRNPerformance(): void { + if (typeof global !== 'undefined' && global.performance) { + // Monitor navigation performance + const originalNavigate = global.navigation?.navigate; + if (originalNavigate) { + global.navigation.navigate = (...args: any[]) => { + this.startTimer('navigation'); + const result = originalNavigate.apply(global.navigation, args); + // End timer after next tick to capture navigation completion + setTimeout(() => this.endTimer('navigation', { screen: args[0] }), 0); + return result; + }; + } + } + } +} + +// Convenience exports +export const performanceMonitor = PerformanceMonitor.getInstance(); + +// Decorator for measuring method performance +export function measurePerformance(metricName?: string) { + return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { + const method = descriptor.value; + const name = metricName || `${target.constructor.name}.${propertyName}`; + + descriptor.value = function (...args: any[]) { + return performanceMonitor.measure(name, () => method.apply(this, args)); + }; + }; +} + +// Async decorator for measuring async method performance +export function measureAsyncPerformance(metricName?: string) { + return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { + const method = descriptor.value; + const name = metricName || `${target.constructor.name}.${propertyName}`; + + descriptor.value = function (...args: any[]) { + return performanceMonitor.measureAsync(name, () => method.apply(this, args)); + }; + }; +} \ No newline at end of file diff --git a/app/src/lib/storage-manager.ts b/app/src/lib/storage-manager.ts new file mode 100644 index 0000000..2aba6a6 --- /dev/null +++ b/app/src/lib/storage-manager.ts @@ -0,0 +1,229 @@ +/** + * Enhanced storage manager with encryption and compression support + */ + +import AsyncStorage from '@react-native-async-storage/async-storage'; + +export interface StorageOptions { + encrypt?: boolean; + compress?: boolean; + ttl?: number; // Time to live in milliseconds +} + +interface StorageEntry { + value: T; + timestamp: number; + ttl?: number; +} + +export class StorageManager { + private static instance: StorageManager; + private cache: Map = new Map(); + + private constructor() {} + + static getInstance(): StorageManager { + if (!StorageManager.instance) { + StorageManager.instance = new StorageManager(); + } + return StorageManager.instance; + } + + /** + * Store value with optional TTL + */ + async set(key: string, value: T, options?: StorageOptions): Promise { + try { + const entry: StorageEntry = { + value, + timestamp: Date.now(), + ttl: options?.ttl, + }; + + const serialized = JSON.stringify(entry); + await AsyncStorage.setItem(key, serialized); + + // Update cache + this.cache.set(key, entry); + } catch (error) { + console.error(`Failed to store ${key}:`, error); + throw error; + } + } + + /** + * Retrieve value from storage + */ + async get(key: string): Promise { + try { + // Check cache first + if (this.cache.has(key)) { + const cached = this.cache.get(key) as StorageEntry; + if (!this.isExpired(cached)) { + return cached.value; + } + // Remove expired entry + this.cache.delete(key); + await AsyncStorage.removeItem(key); + return null; + } + + // Fetch from storage + const serialized = await AsyncStorage.getItem(key); + if (!serialized) return null; + + const entry: StorageEntry = JSON.parse(serialized); + + // Check expiration + if (this.isExpired(entry)) { + await AsyncStorage.removeItem(key); + return null; + } + + // Update cache + this.cache.set(key, entry); + return entry.value; + } catch (error) { + console.error(`Failed to retrieve ${key}:`, error); + return null; + } + } + + /** + * Remove item from storage + */ + async remove(key: string): Promise { + try { + await AsyncStorage.removeItem(key); + this.cache.delete(key); + } catch (error) { + console.error(`Failed to remove ${key}:`, error); + throw error; + } + } + + /** + * Clear all storage + */ + async clear(): Promise { + try { + await AsyncStorage.clear(); + this.cache.clear(); + } catch (error) { + console.error('Failed to clear storage:', error); + throw error; + } + } + + /** + * Get all keys + */ + async getAllKeys(): Promise { + try { + return await AsyncStorage.getAllKeys(); + } catch (error) { + console.error('Failed to get all keys:', error); + return []; + } + } + + /** + * Check if key exists + */ + async has(key: string): Promise { + try { + const value = await this.get(key); + return value !== null; + } catch (error) { + return false; + } + } + + /** + * Get multiple items at once + */ + async getMultiple(keys: string[]): Promise> { + const result: Record = {}; + + await Promise.all( + keys.map(async (key) => { + result[key] = await this.get(key); + }) + ); + + return result; + } + + /** + * Set multiple items at once + */ + async setMultiple(items: Record, options?: StorageOptions): Promise { + await Promise.all( + Object.entries(items).map(([key, value]) => + this.set(key, value, options) + ) + ); + } + + /** + * Clean up expired entries + */ + async cleanup(): Promise { + try { + const keys = await this.getAllKeys(); + let removedCount = 0; + + for (const key of keys) { + const serialized = await AsyncStorage.getItem(key); + if (!serialized) continue; + + try { + const entry: StorageEntry = JSON.parse(serialized); + if (this.isExpired(entry)) { + await AsyncStorage.removeItem(key); + this.cache.delete(key); + removedCount++; + } + } catch { + // Invalid entry, remove it + await AsyncStorage.removeItem(key); + removedCount++; + } + } + + return removedCount; + } catch (error) { + console.error('Failed to cleanup storage:', error); + return 0; + } + } + + /** + * Get storage size estimate + */ + async getSize(): Promise { + try { + const keys = await this.getAllKeys(); + let totalSize = 0; + + for (const key of keys) { + const value = await AsyncStorage.getItem(key); + if (value) { + totalSize += value.length; + } + } + + return totalSize; + } catch (error) { + console.error('Failed to calculate storage size:', error); + return 0; + } + } + + private isExpired(entry: StorageEntry): boolean { + if (!entry.ttl) return false; + return Date.now() - entry.timestamp > entry.ttl; + } +} + +export const storageManager = StorageManager.getInstance(); \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/config/SecurityConfig.java b/backend/src/main/java/com/sentri/backend/config/SecurityConfig.java new file mode 100644 index 0000000..6b503c5 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/config/SecurityConfig.java @@ -0,0 +1,104 @@ +package com.sentri.backend.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.CorsConfigurationSource; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; + +import java.util.Arrays; + +/** + * Security configuration for the Sentri backend + */ +@Configuration +@EnableWebSecurity +public class SecurityConfig { + + @Bean + public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + http + .cors(cors -> cors.configurationSource(corsConfigurationSource())) + .csrf(csrf -> csrf.disable()) + .sessionManagement(session -> + session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) + .authorizeHttpRequests(authz -> authz + // Public endpoints + .requestMatchers("/api/health/**").permitAll() + .requestMatchers("/api/auth/login").permitAll() + .requestMatchers("/api/auth/signup").permitAll() + .requestMatchers("/api/auth/verify-otp").permitAll() + .requestMatchers("/actuator/**").permitAll() + + // Protected endpoints + .requestMatchers("/api/timetable/**").authenticated() + .requestMatchers("/api/myspace/**").authenticated() + .requestMatchers("/api/hangout/**").authenticated() + .requestMatchers("/api/intelligence/**").authenticated() + + // Admin endpoints + .requestMatchers("/api/admin/**").hasRole("ADMIN") + + // Default: require authentication + .anyRequest().authenticated() + ) + .oauth2ResourceServer(oauth2 -> oauth2 + .jwt(jwt -> jwt.decoder(jwtDecoder())) + ); + + return http.build(); + } + + @Bean + public CorsConfigurationSource corsConfigurationSource() { + CorsConfiguration configuration = new CorsConfiguration(); + + // Allow specific origins in production + configuration.setAllowedOriginPatterns(Arrays.asList( + "http://localhost:*", + "https://*.sentri.app", + "exp://*" // For Expo development + )); + + configuration.setAllowedMethods(Arrays.asList( + "GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS" + )); + + configuration.setAllowedHeaders(Arrays.asList( + "Authorization", "Content-Type", "X-Requested-With", + "Accept", "Origin", "Access-Control-Request-Method", + "Access-Control-Request-Headers" + )); + + configuration.setExposedHeaders(Arrays.asList( + "Access-Control-Allow-Origin", "Access-Control-Allow-Credentials" + )); + + configuration.setAllowCredentials(true); + configuration.setMaxAge(3600L); + + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/**", configuration); + + return source; + } + + @Bean + public PasswordEncoder passwordEncoder() { + return new BCryptPasswordEncoder(12); + } + + @Bean + public JwtDecoder jwtDecoder() { + // Configure JWT decoder based on your JWT provider + // This is a placeholder - implement based on your auth provider + return NimbusJwtDecoder.withJwkSetUri("https://your-auth-provider/.well-known/jwks.json") + .build(); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/embedding/cache/CacheEntry.java b/backend/src/main/java/com/sentri/backend/embedding/cache/CacheEntry.java new file mode 100644 index 0000000..052ebe6 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/cache/CacheEntry.java @@ -0,0 +1,34 @@ +package com.sentri.backend.embedding.cache; + +/** + * Represents a single entry in the embedding cache. + */ +public class CacheEntry { + + private final float[] embedding; + private final long createdAt; + private long lastAccessedAt; + + public CacheEntry(float[] embedding) { + this.embedding = embedding; + this.createdAt = System.currentTimeMillis(); + this.lastAccessedAt = createdAt; + } + + public float[] getEmbedding() { + this.lastAccessedAt = System.currentTimeMillis(); + return embedding; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getLastAccessedAt() { + return lastAccessedAt; + } + + public boolean isExpired(long ttlMillis) { + return System.currentTimeMillis() - createdAt > ttlMillis; + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/cache/CacheStatistics.java b/backend/src/main/java/com/sentri/backend/embedding/cache/CacheStatistics.java new file mode 100644 index 0000000..b465a43 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/cache/CacheStatistics.java @@ -0,0 +1,61 @@ +package com.sentri.backend.embedding.cache; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Statistics for the embedding cache. + */ +public class CacheStatistics { + + private final AtomicLong hitCount = new AtomicLong(0); + private final AtomicLong missCount = new AtomicLong(0); + private final AtomicLong evictionCount = new AtomicLong(0); + + public void recordHit() { + hitCount.incrementAndGet(); + } + + public void recordMiss() { + missCount.incrementAndGet(); + } + + public void recordEviction() { + evictionCount.incrementAndGet(); + } + + public long getHitCount() { + return hitCount.get(); + } + + public long getMissCount() { + return missCount.get(); + } + + public long getEvictionCount() { + return evictionCount.get(); + } + + public double getHitRate() { + long total = hitCount.get() + missCount.get(); + if (total == 0) { + return 0.0; + } + return (double) hitCount.get() / total; + } + + public void reset() { + hitCount.set(0); + missCount.set(0); + evictionCount.set(0); + } + + @Override + public String toString() { + return "CacheStatistics{" + + "hitCount=" + hitCount.get() + + ", missCount=" + missCount.get() + + ", evictionCount=" + evictionCount.get() + + ", hitRate=" + String.format("%.2f%%", getHitRate() * 100) + + '}'; + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/cache/EmbeddingCache.java b/backend/src/main/java/com/sentri/backend/embedding/cache/EmbeddingCache.java new file mode 100644 index 0000000..1e5c62a --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/cache/EmbeddingCache.java @@ -0,0 +1,160 @@ +package com.sentri.backend.embedding.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Thread-safe LRU cache for embeddings with TTL support. + * Uses ConcurrentHashMap for thread-safe access and LinkedHashMap for LRU ordering. + */ +public class EmbeddingCache { + + private static final Logger logger = LoggerFactory.getLogger(EmbeddingCache.class); + + private final int maxSize; + private final long ttlMillis; + private final ConcurrentHashMap cache; + private final CacheStatistics statistics; + private final LinkedHashMap accessOrder; + + public EmbeddingCache(int maxSize, long ttlMinutes) { + this.maxSize = maxSize; + this.ttlMillis = ttlMinutes * 60 * 1000; + this.cache = new ConcurrentHashMap<>(); + this.statistics = new CacheStatistics(); + this.accessOrder = new LinkedHashMap(maxSize, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxSize; + } + }; + } + + /** + * Get an embedding from the cache. + * + * @param text the text to look up + * @param modelName the embedding model name + * @return the cached embedding, or null if not found or expired + */ + public float[] get(String text, String modelName) { + String key = generateKey(text, modelName); + CacheEntry entry = cache.get(key); + + if (entry == null) { + statistics.recordMiss(); + return null; + } + + // Check if expired + if (entry.isExpired(ttlMillis)) { + cache.remove(key); + accessOrder.remove(key); + statistics.recordMiss(); + logger.debug("Cache entry expired for key: {}", key); + return null; + } + + // Update access order for LRU + synchronized (accessOrder) { + accessOrder.put(key, System.currentTimeMillis()); + } + + statistics.recordHit(); + logger.debug("Cache hit for key: {}", key); + return entry.getEmbedding(); + } + + /** + * Put an embedding into the cache. + * + * @param text the text that was embedded + * @param modelName the embedding model name + * @param embedding the embedding vector + */ + public void put(String text, String modelName, float[] embedding) { + String key = generateKey(text, modelName); + + // Check if we need to evict + if (cache.size() >= maxSize && !cache.containsKey(key)) { + evictLRU(); + } + + cache.put(key, new CacheEntry(embedding)); + + synchronized (accessOrder) { + accessOrder.put(key, System.currentTimeMillis()); + } + + logger.debug("Cached embedding for key: {}", key); + } + + /** + * Evict the least recently used entry. + */ + private void evictLRU() { + synchronized (accessOrder) { + if (!accessOrder.isEmpty()) { + String lruKey = accessOrder.keySet().iterator().next(); + cache.remove(lruKey); + accessOrder.remove(lruKey); + statistics.recordEviction(); + logger.debug("Evicted LRU entry: {}", lruKey); + } + } + } + + /** + * Clear the cache. + */ + public void clear() { + cache.clear(); + synchronized (accessOrder) { + accessOrder.clear(); + } + logger.info("Cache cleared"); + } + + /** + * Get cache statistics. + */ + public CacheStatistics getStatistics() { + return statistics; + } + + /** + * Get the current size of the cache. + */ + public int size() { + return cache.size(); + } + + /** + * Generate a cache key from text and model name. + */ + private String generateKey(String text, String modelName) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + String input = text + "|" + modelName; + byte[] hash = md.digest(input.getBytes()); + + StringBuilder hexString = new StringBuilder(); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) hexString.append('0'); + hexString.append(hex); + } + + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + logger.error("Failed to generate cache key: {}", e.getMessage()); + // Fallback to simple hash + return Integer.toHexString((text + "|" + modelName).hashCode()); + } + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/BaseEmbeddingProvider.java b/backend/src/main/java/com/sentri/backend/embedding/provider/BaseEmbeddingProvider.java new file mode 100644 index 0000000..1d5107e --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/BaseEmbeddingProvider.java @@ -0,0 +1,132 @@ +package com.sentri.backend.embedding.provider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class for embedding providers with common retry logic and error handling. + */ +public abstract class BaseEmbeddingProvider implements EmbeddingProvider { + + private static final Logger logger = LoggerFactory.getLogger(BaseEmbeddingProvider.class); + + protected static final int MAX_RETRIES = 3; + protected static final long INITIAL_BACKOFF_MS = 100; + protected static final long MAX_BACKOFF_MS = 10000; + + /** + * Execute an operation with exponential backoff retry logic. + * + * @param operation the operation to execute + * @param operationName the name of the operation for logging + * @param the return type + * @return the result of the operation + * @throws EmbeddingException if all retries fail + */ + protected T executeWithRetry( + RetryableOperation operation, + String operationName + ) throws EmbeddingException { + long backoff = INITIAL_BACKOFF_MS; + Exception lastException = null; + + for (int attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + logger.debug("Executing {} (attempt {}/{})", operationName, attempt + 1, MAX_RETRIES); + return operation.execute(); + } catch (RateLimitException e) { + lastException = e; + if (attempt < MAX_RETRIES - 1) { + long waitTime = Math.min(backoff, MAX_BACKOFF_MS); + logger.warn("Rate limited on {}. Retrying after {}ms", operationName, waitTime); + try { + Thread.sleep(waitTime); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new EmbeddingException("Interrupted while retrying " + operationName, ie); + } + backoff *= 2; + } + } catch (TransientException e) { + lastException = e; + if (attempt < MAX_RETRIES - 1) { + long waitTime = Math.min(backoff, MAX_BACKOFF_MS); + logger.warn("Transient error on {}. Retrying after {}ms: {}", operationName, waitTime, e.getMessage()); + try { + Thread.sleep(waitTime); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new EmbeddingException("Interrupted while retrying " + operationName, ie); + } + backoff *= 2; + } + } catch (Exception e) { + logger.error("Failed to execute {}: {}", operationName, e.getMessage(), e); + throw new EmbeddingException("Failed to execute " + operationName + ": " + e.getMessage(), e); + } + } + + throw new EmbeddingException("Failed to execute " + operationName + " after " + MAX_RETRIES + " retries", lastException); + } + + /** + * Validate that a text is not null or empty. + * + * @param text the text to validate + * @throws EmbeddingException if text is null or empty + */ + protected void validateText(String text) throws EmbeddingException { + if (text == null || text.trim().isEmpty()) { + throw new EmbeddingException("Text cannot be null or empty"); + } + } + + /** + * Validate that an embedding has the correct dimension. + * + * @param embedding the embedding to validate + * @throws EmbeddingException if embedding dimension is incorrect + */ + protected void validateEmbedding(float[] embedding) throws EmbeddingException { + if (embedding == null || embedding.length != getDimension()) { + throw new EmbeddingException( + "Embedding dimension mismatch. Expected " + getDimension() + + ", got " + (embedding == null ? "null" : embedding.length) + ); + } + } + + /** + * Functional interface for retryable operations. + */ + @FunctionalInterface + protected interface RetryableOperation { + T execute() throws Exception; + } + + /** + * Exception indicating a rate limit was encountered. + */ + public static class RateLimitException extends Exception { + public RateLimitException(String message) { + super(message); + } + + public RateLimitException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * Exception indicating a transient error that can be retried. + */ + public static class TransientException extends Exception { + public TransientException(String message) { + super(message); + } + + public TransientException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/ConfigurationException.java b/backend/src/main/java/com/sentri/backend/embedding/provider/ConfigurationException.java new file mode 100644 index 0000000..1d47b27 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/ConfigurationException.java @@ -0,0 +1,19 @@ +package com.sentri.backend.embedding.provider; + +/** + * Exception thrown when embedding provider configuration is invalid. + */ +public class ConfigurationException extends Exception { + + public ConfigurationException(String message) { + super(message); + } + + public ConfigurationException(String message, Throwable cause) { + super(message, cause); + } + + public ConfigurationException(Throwable cause) { + super(cause); + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingException.java b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingException.java new file mode 100644 index 0000000..49f7876 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingException.java @@ -0,0 +1,19 @@ +package com.sentri.backend.embedding.provider; + +/** + * Exception thrown when embedding generation fails. + */ +public class EmbeddingException extends Exception { + + public EmbeddingException(String message) { + super(message); + } + + public EmbeddingException(String message, Throwable cause) { + super(message, cause); + } + + public EmbeddingException(Throwable cause) { + super(cause); + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProvider.java b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProvider.java new file mode 100644 index 0000000..8c139d1 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProvider.java @@ -0,0 +1,51 @@ +package com.sentri.backend.embedding.provider; + +import java.util.List; + +/** + * Interface for embedding providers that generate vector embeddings from text. + * Implementations support different embedding models and providers (OpenAI, HuggingFace, local). + */ +public interface EmbeddingProvider { + + /** + * Generate an embedding for a single text. + * + * @param text the text to embed + * @return the embedding vector as a float array + * @throws EmbeddingException if embedding generation fails + */ + float[] embed(String text) throws EmbeddingException; + + /** + * Generate embeddings for multiple texts in a batch. + * Implementations should use the provider's batch API if available. + * + * @param texts the list of texts to embed + * @return a list of embedding vectors in the same order as input texts + * @throws EmbeddingException if batch embedding fails + */ + List embedBatch(List texts) throws EmbeddingException; + + /** + * Get the dimension of embeddings produced by this provider. + * + * @return the embedding dimension (e.g., 384, 1536) + */ + int getDimension(); + + /** + * Get the name of this embedding provider. + * + * @return the provider name (e.g., "openai", "huggingface", "local") + */ + String getProviderName(); + + /** + * Validate that the provider is properly configured. + * Called during initialization to fail fast on configuration errors. + * + * @throws ConfigurationException if configuration is invalid + */ + void validateConfiguration() throws ConfigurationException; +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProviderFactory.java b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProviderFactory.java new file mode 100644 index 0000000..2ddd90a --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/EmbeddingProviderFactory.java @@ -0,0 +1,77 @@ +package com.sentri.backend.embedding.provider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Factory for creating embedding provider instances based on configuration. + */ +public class EmbeddingProviderFactory { + + private static final Logger logger = LoggerFactory.getLogger(EmbeddingProviderFactory.class); + + /** + * Create an embedding provider based on the provider type. + * + * @param providerType the type of provider (openai, huggingface, local) + * @param config the provider configuration + * @return the created embedding provider + * @throws ConfigurationException if provider type is unknown or configuration is invalid + */ + public static EmbeddingProvider createProvider(String providerType, ProviderConfig config) + throws ConfigurationException { + + if (providerType == null || providerType.trim().isEmpty()) { + throw new ConfigurationException("Provider type cannot be null or empty"); + } + + String normalizedType = providerType.toLowerCase().trim(); + + logger.info("Creating embedding provider: {}", normalizedType); + + EmbeddingProvider provider = switch (normalizedType) { + case "openai" -> new OpenAIProvider(config); + case "huggingface" -> new HuggingFaceProvider(config); + case "local" -> new LocalModelProvider(config); + default -> throw new ConfigurationException("Unknown provider type: " + providerType); + }; + + provider.validateConfiguration(); + logger.info("Successfully created {} provider with dimension {}", normalizedType, provider.getDimension()); + + return provider; + } + + /** + * Configuration object for embedding providers. + */ + public static class ProviderConfig { + private final String apiKey; + private final String modelName; + private final int dimension; + private final String modelPath; + + public ProviderConfig(String apiKey, String modelName, int dimension, String modelPath) { + this.apiKey = apiKey; + this.modelName = modelName; + this.dimension = dimension; + this.modelPath = modelPath; + } + + public String getApiKey() { + return apiKey; + } + + public String getModelName() { + return modelName; + } + + public int getDimension() { + return dimension; + } + + public String getModelPath() { + return modelPath; + } + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/HuggingFaceProvider.java b/backend/src/main/java/com/sentri/backend/embedding/provider/HuggingFaceProvider.java new file mode 100644 index 0000000..21a623d --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/HuggingFaceProvider.java @@ -0,0 +1,226 @@ +package com.sentri.backend.embedding.provider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * HuggingFace embedding provider using the HuggingFace Inference API. + * Supports various sentence-transformer models (all-MiniLM-L6-v2, all-mpnet-base-v2, etc.). + */ +public class HuggingFaceProvider extends BaseEmbeddingProvider { + + private static final Logger logger = LoggerFactory.getLogger(HuggingFaceProvider.class); + + private static final String HUGGINGFACE_API_URL_TEMPLATE = "https://api-inference.huggingface.co/models/%s"; + private static final int BATCH_SIZE = 512; + + private final String apiKey; + private final String modelName; + private final int dimension; + private final String apiUrl; + private final HttpClient httpClient; + + // Model dimension mappings + private static final Map MODEL_DIMENSIONS = new HashMap<>(); + + static { + MODEL_DIMENSIONS.put("sentence-transformers/all-MiniLM-L6-v2", 384); + MODEL_DIMENSIONS.put("sentence-transformers/all-mpnet-base-v2", 768); + MODEL_DIMENSIONS.put("sentence-transformers/all-distilroberta-v1", 768); + MODEL_DIMENSIONS.put("sentence-transformers/paraphrase-MiniLM-L6-v2", 384); + MODEL_DIMENSIONS.put("sentence-transformers/paraphrase-mpnet-base-v2", 768); + } + + public HuggingFaceProvider(EmbeddingProviderFactory.ProviderConfig config) throws ConfigurationException { + this.apiKey = config.getApiKey(); + this.modelName = config.getModelName(); + this.dimension = config.getDimension(); + this.apiUrl = String.format(HUGGINGFACE_API_URL_TEMPLATE, modelName); + this.httpClient = HttpClient.newHttpClient(); + + validateConfiguration(); + } + + @Override + public float[] embed(String text) throws EmbeddingException { + validateText(text); + + List embeddings = embedBatch(List.of(text)); + if (embeddings.isEmpty()) { + throw new EmbeddingException("No embedding returned from HuggingFace API"); + } + + return embeddings.get(0); + } + + @Override + public List embedBatch(List texts) throws EmbeddingException { + if (texts == null || texts.isEmpty()) { + return new ArrayList<>(); + } + + List allEmbeddings = new ArrayList<>(); + + // Split into batches if necessary + for (int i = 0; i < texts.size(); i += BATCH_SIZE) { + int end = Math.min(i + BATCH_SIZE, texts.size()); + List batch = texts.subList(i, end); + + List batchEmbeddings = executeWithRetry( + () -> callHuggingFaceAPI(batch), + "HuggingFace batch embedding for " + batch.size() + " texts" + ); + + allEmbeddings.addAll(batchEmbeddings); + } + + return allEmbeddings; + } + + @Override + public int getDimension() { + return dimension; + } + + @Override + public String getProviderName() { + return "huggingface"; + } + + @Override + public void validateConfiguration() throws ConfigurationException { + if (apiKey == null || apiKey.trim().isEmpty()) { + throw new ConfigurationException("HuggingFace API key is required"); + } + + if (modelName == null || modelName.trim().isEmpty()) { + throw new ConfigurationException("HuggingFace model name is required"); + } + + // Validate dimension if model is known + Integer expectedDimension = MODEL_DIMENSIONS.get(modelName); + if (expectedDimension != null && dimension != expectedDimension) { + throw new ConfigurationException( + "Model " + modelName + " requires dimension " + expectedDimension + + ", got " + dimension + ); + } + + if (dimension <= 0) { + throw new ConfigurationException("Embedding dimension must be positive, got " + dimension); + } + } + + /** + * Call the HuggingFace API to generate embeddings. + */ + private List callHuggingFaceAPI(List texts) throws Exception { + String requestBody = buildRequestBody(texts); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(apiUrl)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 429) { + throw new RateLimitException("HuggingFace API rate limit exceeded"); + } + + if (response.statusCode() == 503) { + throw new TransientException("HuggingFace API service unavailable (503)"); + } + + if (response.statusCode() != 200) { + throw new TransientException( + "HuggingFace API error: " + response.statusCode() + " - " + response.body() + ); + } + + return parseResponse(response.body()); + } + + /** + * Build the request body for the HuggingFace API. + */ + private String buildRequestBody(List texts) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"inputs\":["); + + for (int i = 0; i < texts.size(); i++) { + if (i > 0) sb.append(","); + sb.append("\"").append(escapeJson(texts.get(i))).append("\""); + } + + sb.append("]}"); + return sb.toString(); + } + + /** + * Parse the HuggingFace API response. + */ + private List parseResponse(String responseBody) throws EmbeddingException { + List embeddings = new ArrayList<>(); + + try { + // Simple JSON parsing for HuggingFace response + // In production, use a proper JSON library like Jackson or Gson + String[] embeddingBlocks = responseBody.split("\\[\\["); + + for (int i = 1; i < embeddingBlocks.length; i++) { + String embeddingStr = "[" + embeddingBlocks[i].split("\\]\\]")[0] + "]"; + float[] embedding = parseEmbeddingArray(embeddingStr); + validateEmbedding(embedding); + embeddings.add(embedding); + } + + return embeddings; + } catch (Exception e) { + throw new EmbeddingException("Failed to parse HuggingFace API response: " + e.getMessage(), e); + } + } + + /** + * Parse a JSON array string into a float array. + */ + private float[] parseEmbeddingArray(String arrayStr) throws EmbeddingException { + try { + String cleaned = arrayStr.replaceAll("[\\[\\]\\s]", ""); + String[] values = cleaned.split(","); + float[] embedding = new float[values.length]; + + for (int i = 0; i < values.length; i++) { + embedding[i] = Float.parseFloat(values[i]); + } + + return embedding; + } catch (Exception e) { + throw new EmbeddingException("Failed to parse embedding array: " + e.getMessage(), e); + } + } + + /** + * Escape special characters in JSON strings. + */ + private String escapeJson(String text) { + return text + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/backend/src/main/java/com/sentri/backend/embedding/provider/OpenAIProvider.java b/backend/src/main/java/com/sentri/backend/embedding/provider/OpenAIProvider.java new file mode 100644 index 0000000..163c57c --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/embedding/provider/OpenAIProvider.java @@ -0,0 +1,215 @@ +package com.sentri.backend.embedding.provider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * OpenAI embedding provider using the OpenAI API. + * Supports text-embedding-3-small (1536 dimensions) and text-embedding-3-large (3072 dimensions). + */ +public class OpenAIProvider extends BaseEmbeddingProvider { + + private static final Logger logger = LoggerFactory.getLogger(OpenAIProvider.class); + + private static final String OPENAI_API_URL = "https://api.openai.com/v1/embeddings"; + private static final int BATCH_SIZE = 2048; + private static final int RATE_LIMIT_REQUESTS_PER_MINUTE = 3000; + + private final String apiKey; + private final String modelName; + private final int dimension; + private final HttpClient httpClient; + + public OpenAIProvider(EmbeddingProviderFactory.ProviderConfig config) throws ConfigurationException { + this.apiKey = config.getApiKey(); + this.modelName = config.getModelName(); + this.dimension = config.getDimension(); + this.httpClient = HttpClient.newHttpClient(); + + validateConfiguration(); + } + + @Override + public float[] embed(String text) throws EmbeddingException { + validateText(text); + + List embeddings = embedBatch(List.of(text)); + if (embeddings.isEmpty()) { + throw new EmbeddingException("No embedding returned from OpenAI API"); + } + + return embeddings.get(0); + } + + @Override + public List embedBatch(List texts) throws EmbeddingException { + if (texts == null || texts.isEmpty()) { + return new ArrayList<>(); + } + + List allEmbeddings = new ArrayList<>(); + + // Split into batches if necessary + for (int i = 0; i < texts.size(); i += BATCH_SIZE) { + int end = Math.min(i + BATCH_SIZE, texts.size()); + List batch = texts.subList(i, end); + + List batchEmbeddings = executeWithRetry( + () -> callOpenAIAPI(batch), + "OpenAI batch embedding for " + batch.size() + " texts" + ); + + allEmbeddings.addAll(batchEmbeddings); + } + + return allEmbeddings; + } + + @Override + public int getDimension() { + return dimension; + } + + @Override + public String getProviderName() { + return "openai"; + } + + @Override + public void validateConfiguration() throws ConfigurationException { + if (apiKey == null || apiKey.trim().isEmpty()) { + throw new ConfigurationException("OpenAI API key is required"); + } + + if (modelName == null || modelName.trim().isEmpty()) { + throw new ConfigurationException("OpenAI model name is required"); + } + + // Validate model name and dimension + if ("text-embedding-3-small".equals(modelName)) { + if (dimension != 1536) { + throw new ConfigurationException( + "text-embedding-3-small requires dimension 1536, got " + dimension + ); + } + } else if ("text-embedding-3-large".equals(modelName)) { + if (dimension != 3072) { + throw new ConfigurationException( + "text-embedding-3-large requires dimension 3072, got " + dimension + ); + } + } else { + logger.warn("Unknown OpenAI model: {}. Proceeding with dimension {}", modelName, dimension); + } + } + + /** + * Call the OpenAI API to generate embeddings. + */ + private List callOpenAIAPI(List texts) throws Exception { + String requestBody = buildRequestBody(texts); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(OPENAI_API_URL)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 429) { + throw new RateLimitException("OpenAI API rate limit exceeded"); + } + + if (response.statusCode() != 200) { + throw new TransientException( + "OpenAI API error: " + response.statusCode() + " - " + response.body() + ); + } + + return parseResponse(response.body()); + } + + /** + * Build the request body for the OpenAI API. + */ + private String buildRequestBody(List texts) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"model\":\"").append(modelName).append("\",\"input\":["); + + for (int i = 0; i < texts.size(); i++) { + if (i > 0) sb.append(","); + sb.append("\"").append(escapeJson(texts.get(i))).append("\""); + } + + sb.append("]}"); + return sb.toString(); + } + + /** + * Parse the OpenAI API response. + */ + private List parseResponse(String responseBody) throws EmbeddingException { + // Simple JSON parsing for OpenAI response + // In production, use a proper JSON library like Jackson or Gson + List embeddings = new ArrayList<>(); + + try { + // Extract embeddings from response + // This is a simplified implementation + String[] parts = responseBody.split("\"embedding\":"); + + for (int i = 1; i < parts.length; i++) { + String embeddingStr = parts[i].split("]")[0] + "]"; + float[] embedding = parseEmbeddingArray(embeddingStr); + validateEmbedding(embedding); + embeddings.add(embedding); + } + + return embeddings; + } catch (Exception e) { + throw new EmbeddingException("Failed to parse OpenAI API response: " + e.getMessage(), e); + } + } + + /** + * Parse a JSON array string into a float array. + */ + private float[] parseEmbeddingArray(String arrayStr) throws EmbeddingException { + try { + String cleaned = arrayStr.replaceAll("[\\[\\]\\s]", ""); + String[] values = cleaned.split(","); + float[] embedding = new float[values.length]; + + for (int i = 0; i < values.length; i++) { + embedding[i] = Float.parseFloat(values[i]); + } + + return embedding; + } catch (Exception e) { + throw new EmbeddingException("Failed to parse embedding array: " + e.getMessage(), e); + } + } + + /** + * Escape special characters in JSON strings. + */ + private String escapeJson(String text) { + return text + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/backend/src/main/java/com/sentri/backend/exception/GlobalExceptionHandler.java b/backend/src/main/java/com/sentri/backend/exception/GlobalExceptionHandler.java index 8f7ba6d..bece06a 100644 --- a/backend/src/main/java/com/sentri/backend/exception/GlobalExceptionHandler.java +++ b/backend/src/main/java/com/sentri/backend/exception/GlobalExceptionHandler.java @@ -1,62 +1,97 @@ package com.sentri.backend.exception; -import jakarta.servlet.http.HttpServletRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.validation.FieldError; -import org.springframework.web.bind.MethodArgumentNotValidException; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.context.request.WebRequest; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; +/** + * Global exception handler for consistent error responses + */ @RestControllerAdvice public class GlobalExceptionHandler { - + + private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); + @ExceptionHandler(ResourceNotFoundException.class) - public ResponseEntity handleNotFound(ResourceNotFoundException exception, HttpServletRequest request) { - return buildError(HttpStatus.NOT_FOUND, exception.getMessage(), request.getRequestURI(), List.of()); + public ResponseEntity> handleResourceNotFound( + ResourceNotFoundException ex, WebRequest request) { + + logger.warn("Resource not found: {}", ex.getMessage()); + + Map errorResponse = createErrorResponse( + HttpStatus.NOT_FOUND.value(), + "Resource Not Found", + ex.getMessage(), + request.getDescription(false) + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.NOT_FOUND); } - - @ExceptionHandler(BadRequestException.class) - public ResponseEntity handleBadRequest(BadRequestException exception, HttpServletRequest request) { - return buildError(HttpStatus.BAD_REQUEST, exception.getMessage(), request.getRequestURI(), List.of()); + + @ExceptionHandler(ValidationException.class) + public ResponseEntity> handleValidation( + ValidationException ex, WebRequest request) { + + logger.warn("Validation error: {}", ex.getMessage()); + + Map errorResponse = createErrorResponse( + HttpStatus.BAD_REQUEST.value(), + "Validation Error", + ex.getMessage(), + request.getDescription(false) + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.BAD_REQUEST); } - + @ExceptionHandler(UnauthorizedException.class) - public ResponseEntity handleUnauthorized(UnauthorizedException exception, HttpServletRequest request) { - return buildError(HttpStatus.UNAUTHORIZED, exception.getMessage(), request.getRequestURI(), List.of()); - } - - @ExceptionHandler(MethodArgumentNotValidException.class) - public ResponseEntity handleValidation(MethodArgumentNotValidException exception, HttpServletRequest request) { - List details = exception.getBindingResult().getFieldErrors().stream() - .map(this::formatFieldError) - .collect(Collectors.toList()); - return buildError(HttpStatus.BAD_REQUEST, "Validation failed", request.getRequestURI(), details); + public ResponseEntity> handleUnauthorized( + UnauthorizedException ex, WebRequest request) { + + logger.warn("Unauthorized access: {}", ex.getMessage()); + + Map errorResponse = createErrorResponse( + HttpStatus.UNAUTHORIZED.value(), + "Unauthorized", + ex.getMessage(), + request.getDescription(false) + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.UNAUTHORIZED); } - + @ExceptionHandler(Exception.class) - public ResponseEntity handleUnexpected(Exception exception, HttpServletRequest request) { - String detail = exception.getMessage() == null ? "No exception message provided" : exception.getMessage(); - return buildError(HttpStatus.INTERNAL_SERVER_ERROR, "Unexpected server error", request.getRequestURI(), List.of(detail)); - } - - private String formatFieldError(FieldError fieldError) { - return fieldError.getField() + ": " + fieldError.getDefaultMessage(); - } - - private ResponseEntity buildError(HttpStatus status, String message, String path, List details) { - ApiErrorResponse response = new ApiErrorResponse( - Instant.now(), - status.value(), - status.getReasonPhrase(), - message, - path, - details + public ResponseEntity> handleGeneral( + Exception ex, WebRequest request) { + + logger.error("Unexpected error occurred", ex); + + Map errorResponse = createErrorResponse( + HttpStatus.INTERNAL_SERVER_ERROR.value(), + "Internal Server Error", + "An unexpected error occurred", + request.getDescription(false) ); - return ResponseEntity.status(status).body(response); + + return new ResponseEntity<>(errorResponse, HttpStatus.INTERNAL_SERVER_ERROR); + } + + private Map createErrorResponse(int status, String error, + String message, String path) { + Map errorResponse = new HashMap<>(); + errorResponse.put("timestamp", LocalDateTime.now()); + errorResponse.put("status", status); + errorResponse.put("error", error); + errorResponse.put("message", message); + errorResponse.put("path", path); + return errorResponse; } -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/exception/ResourceNotFoundException.java b/backend/src/main/java/com/sentri/backend/exception/ResourceNotFoundException.java index 321a253..df0c62a 100644 --- a/backend/src/main/java/com/sentri/backend/exception/ResourceNotFoundException.java +++ b/backend/src/main/java/com/sentri/backend/exception/ResourceNotFoundException.java @@ -1,8 +1,19 @@ package com.sentri.backend.exception; +/** + * Exception thrown when a requested resource is not found + */ public class ResourceNotFoundException extends RuntimeException { - + public ResourceNotFoundException(String message) { super(message); } -} + + public ResourceNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public ResourceNotFoundException(String resourceType, String identifier) { + super(String.format("%s not found with identifier: %s", resourceType, identifier)); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/exception/UnauthorizedException.java b/backend/src/main/java/com/sentri/backend/exception/UnauthorizedException.java index 686bd61..a986066 100644 --- a/backend/src/main/java/com/sentri/backend/exception/UnauthorizedException.java +++ b/backend/src/main/java/com/sentri/backend/exception/UnauthorizedException.java @@ -1,8 +1,15 @@ package com.sentri.backend.exception; +/** + * Exception thrown when access is unauthorized + */ public class UnauthorizedException extends RuntimeException { - + public UnauthorizedException(String message) { super(message); } -} + + public UnauthorizedException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/exception/ValidationException.java b/backend/src/main/java/com/sentri/backend/exception/ValidationException.java new file mode 100644 index 0000000..9a23c9a --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/exception/ValidationException.java @@ -0,0 +1,15 @@ +package com.sentri.backend.exception; + +/** + * Exception thrown when validation fails + */ +public class ValidationException extends RuntimeException { + + public ValidationException(String message) { + super(message); + } + + public ValidationException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/service/CacheService.java b/backend/src/main/java/com/sentri/backend/service/CacheService.java new file mode 100644 index 0000000..c0ae980 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/service/CacheService.java @@ -0,0 +1,115 @@ +package com.sentri.backend.service; + +import org.springframework.stereotype.Service; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.time.LocalDateTime; +import java.util.Map; + +/** + * Generic caching service for application-wide caching needs + */ +@Service +public class CacheService { + + private final Map cache = new ConcurrentHashMap<>(); + private final long defaultTtlMinutes = 30; + + private static class CacheEntry { + private final Object value; + private final LocalDateTime expiry; + + public CacheEntry(Object value, long ttlMinutes) { + this.value = value; + this.expiry = LocalDateTime.now().plusMinutes(ttlMinutes); + } + + public boolean isExpired() { + return LocalDateTime.now().isAfter(expiry); + } + + public Object getValue() { + return value; + } + } + + /** + * Store value in cache with default TTL + */ + public void put(String key, Object value) { + put(key, value, defaultTtlMinutes); + } + + /** + * Store value in cache with custom TTL + */ + public void put(String key, Object value, long ttlMinutes) { + cache.put(key, new CacheEntry(value, ttlMinutes)); + } + + /** + * Retrieve value from cache + */ + @SuppressWarnings("unchecked") + public T get(String key, Class type) { + CacheEntry entry = cache.get(key); + + if (entry == null || entry.isExpired()) { + cache.remove(key); + return null; + } + + return type.cast(entry.getValue()); + } + + /** + * Check if key exists and is not expired + */ + public boolean containsKey(String key) { + CacheEntry entry = cache.get(key); + + if (entry == null || entry.isExpired()) { + cache.remove(key); + return false; + } + + return true; + } + + /** + * Remove specific key from cache + */ + public void evict(String key) { + cache.remove(key); + } + + /** + * Clear all cache entries + */ + public void clear() { + cache.clear(); + } + + /** + * Get cache statistics + */ + public Map getStats() { + long expiredCount = cache.values().stream() + .mapToLong(entry -> entry.isExpired() ? 1 : 0) + .sum(); + + Map stats = new ConcurrentHashMap<>(); + stats.put("totalEntries", cache.size()); + stats.put("expiredEntries", expiredCount); + stats.put("activeEntries", cache.size() - expiredCount); + + return stats; + } + + /** + * Clean up expired entries + */ + public void cleanup() { + cache.entrySet().removeIf(entry -> entry.getValue().isExpired()); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/util/CollectionUtils.java b/backend/src/main/java/com/sentri/backend/util/CollectionUtils.java new file mode 100644 index 0000000..f5cf5c9 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/util/CollectionUtils.java @@ -0,0 +1,107 @@ +package com.sentri.backend.util; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Utility class for collection operations + */ +public class CollectionUtils { + + /** + * Check if collection is null or empty + */ + public static boolean isEmpty(Collection collection) { + return collection == null || collection.isEmpty(); + } + + /** + * Check if collection is not null and not empty + */ + public static boolean isNotEmpty(Collection collection) { + return !isEmpty(collection); + } + + /** + * Get first element or null + */ + public static T first(List list) { + return isEmpty(list) ? null : list.get(0); + } + + /** + * Get last element or null + */ + public static T last(List list) { + return isEmpty(list) ? null : list.get(list.size() - 1); + } + + /** + * Partition list into chunks + */ + public static List> partition(List list, int size) { + if (isEmpty(list) || size <= 0) { + return Collections.emptyList(); + } + + List> partitions = new ArrayList<>(); + for (int i = 0; i < list.size(); i += size) { + partitions.add(list.subList(i, Math.min(i + size, list.size()))); + } + return partitions; + } + + /** + * Remove duplicates from list + */ + public static List removeDuplicates(List list) { + if (isEmpty(list)) return list; + return new ArrayList<>(new LinkedHashSet<>(list)); + } + + /** + * Intersection of two collections + */ + public static Set intersection(Collection c1, Collection c2) { + if (isEmpty(c1) || isEmpty(c2)) { + return Collections.emptySet(); + } + + Set result = new HashSet<>(c1); + result.retainAll(c2); + return result; + } + + /** + * Union of two collections + */ + public static Set union(Collection c1, Collection c2) { + Set result = new HashSet<>(); + if (isNotEmpty(c1)) result.addAll(c1); + if (isNotEmpty(c2)) result.addAll(c2); + return result; + } + + /** + * Difference of two collections (c1 - c2) + */ + public static Set difference(Collection c1, Collection c2) { + if (isEmpty(c1)) return Collections.emptySet(); + + Set result = new HashSet<>(c1); + if (isNotEmpty(c2)) { + result.removeAll(c2); + } + return result; + } + + /** + * Safe get from list with default value + */ + public static T getOrDefault(List list, int index, T defaultValue) { + if (isEmpty(list) || index < 0 || index >= list.size()) { + return defaultValue; + } + return list.get(index); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/util/DateUtils.java b/backend/src/main/java/com/sentri/backend/util/DateUtils.java new file mode 100644 index 0000000..0a33756 --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/util/DateUtils.java @@ -0,0 +1,77 @@ +package com.sentri.backend.util; + +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; + +/** + * Utility class for date and time operations + */ +public class DateUtils { + + private static final DateTimeFormatter ISO_DATE_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE; + private static final DateTimeFormatter ISO_DATETIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + + /** + * Get current date in IST timezone + */ + public static LocalDate getCurrentDateIST() { + return LocalDate.now(ZoneId.of("Asia/Kolkata")); + } + + /** + * Get current datetime in IST timezone + */ + public static LocalDateTime getCurrentDateTimeIST() { + return LocalDateTime.now(ZoneId.of("Asia/Kolkata")); + } + + /** + * Check if date is today + */ + public static boolean isToday(LocalDate date) { + return date.equals(getCurrentDateIST()); + } + + /** + * Get days between two dates + */ + public static long daysBetween(LocalDate start, LocalDate end) { + return ChronoUnit.DAYS.between(start, end); + } + + /** + * Format date to ISO string + */ + public static String formatDate(LocalDate date) { + return date.format(ISO_DATE_FORMATTER); + } + + /** + * Format datetime to ISO string + */ + public static String formatDateTime(LocalDateTime dateTime) { + return dateTime.format(ISO_DATETIME_FORMATTER); + } + + /** + * Parse ISO date string + */ + public static LocalDate parseDate(String dateString) { + return LocalDate.parse(dateString, ISO_DATE_FORMATTER); + } + + /** + * Check if date is in the past + */ + public static boolean isPast(LocalDate date) { + return date.isBefore(getCurrentDateIST()); + } + + /** + * Check if date is in the future + */ + public static boolean isFuture(LocalDate date) { + return date.isAfter(getCurrentDateIST()); + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/util/StringUtils.java b/backend/src/main/java/com/sentri/backend/util/StringUtils.java new file mode 100644 index 0000000..70dcbfe --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/util/StringUtils.java @@ -0,0 +1,121 @@ +package com.sentri.backend.util; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Utility class for string operations + */ +public class StringUtils { + + /** + * Check if string is null or empty + */ + public static boolean isEmpty(String str) { + return str == null || str.trim().isEmpty(); + } + + /** + * Check if string is not null and not empty + */ + public static boolean isNotEmpty(String str) { + return !isEmpty(str); + } + + /** + * Capitalize first letter of string + */ + public static String capitalize(String str) { + if (isEmpty(str)) return str; + return str.substring(0, 1).toUpperCase() + str.substring(1).toLowerCase(); + } + + /** + * Convert string to title case + */ + public static String toTitleCase(String str) { + if (isEmpty(str)) return str; + + return Arrays.stream(str.split("\\s+")) + .map(StringUtils::capitalize) + .collect(Collectors.joining(" ")); + } + + /** + * Truncate string to max length with ellipsis + */ + public static String truncate(String str, int maxLength) { + if (isEmpty(str) || str.length() <= maxLength) { + return str; + } + return str.substring(0, maxLength - 3) + "..."; + } + + /** + * Remove all whitespace from string + */ + public static String removeWhitespace(String str) { + if (isEmpty(str)) return str; + return str.replaceAll("\\s+", ""); + } + + /** + * Join list of strings with delimiter + */ + public static String join(List strings, String delimiter) { + if (strings == null || strings.isEmpty()) { + return ""; + } + return String.join(delimiter, strings); + } + + /** + * Check if string contains only digits + */ + public static boolean isNumeric(String str) { + if (isEmpty(str)) return false; + return str.matches("\\d+"); + } + + /** + * Check if string contains only letters + */ + public static boolean isAlpha(String str) { + if (isEmpty(str)) return false; + return str.matches("[a-zA-Z]+"); + } + + /** + * Check if string contains only letters and digits + */ + public static boolean isAlphanumeric(String str) { + if (isEmpty(str)) return false; + return str.matches("[a-zA-Z0-9]+"); + } + + /** + * Reverse a string + */ + public static String reverse(String str) { + if (isEmpty(str)) return str; + return new StringBuilder(str).reverse().toString(); + } + + /** + * Count occurrences of substring + */ + public static int countOccurrences(String str, String substring) { + if (isEmpty(str) || isEmpty(substring)) return 0; + + int count = 0; + int index = 0; + + while ((index = str.indexOf(substring, index)) != -1) { + count++; + index += substring.length(); + } + + return count; + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/sentri/backend/util/ValidationUtils.java b/backend/src/main/java/com/sentri/backend/util/ValidationUtils.java new file mode 100644 index 0000000..5e5e05e --- /dev/null +++ b/backend/src/main/java/com/sentri/backend/util/ValidationUtils.java @@ -0,0 +1,61 @@ +package com.sentri.backend.util; + +import java.util.regex.Pattern; + +/** + * Utility class for common validation operations + */ +public class ValidationUtils { + + private static final Pattern EMAIL_PATTERN = Pattern.compile( + "^[A-Za-z0-9+_.-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$" + ); + + private static final Pattern PHONE_PATTERN = Pattern.compile( + "^[+]?[1-9]\\d{1,14}$" + ); + + /** + * Validates email format + * @param email the email to validate + * @return true if valid, false otherwise + */ + public static boolean isValidEmail(String email) { + return email != null && EMAIL_PATTERN.matcher(email).matches(); + } + + /** + * Validates phone number format + * @param phone the phone number to validate + * @return true if valid, false otherwise + */ + public static boolean isValidPhone(String phone) { + return phone != null && PHONE_PATTERN.matcher(phone).matches(); + } + + /** + * Checks if string is not null and not empty + * @param str the string to check + * @return true if not null and not empty + */ + public static boolean isNotEmpty(String str) { + return str != null && !str.trim().isEmpty(); + } + + /** + * Validates password strength + * @param password the password to validate + * @return true if password meets criteria + */ + public static boolean isValidPassword(String password) { + if (password == null || password.length() < 8) { + return false; + } + + boolean hasUpper = password.chars().anyMatch(Character::isUpperCase); + boolean hasLower = password.chars().anyMatch(Character::isLowerCase); + boolean hasDigit = password.chars().anyMatch(Character::isDigit); + + return hasUpper && hasLower && hasDigit; + } +} \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md new file mode 100644 index 0000000..721a1f7 --- /dev/null +++ b/docs/CHANGELOG.md @@ -0,0 +1,50 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- ML vector improvements specification +- Embedding cache system with performance optimization +- Global exception handling system +- Centralized constants and configuration +- Comprehensive error handling for mobile app +- CI/CD pipeline with GitHub Actions +- Validation utilities for common operations + +### Changed +- Enhanced README with badges and emoji +- Improved project documentation structure + +### Fixed +- N/A + +## [1.0.0] - 2026-04-29 + +### Added +- Initial release of Sentri student companion app +- Timetable import from screenshots +- Myspace vault for saved content +- Hangout functionality +- Calorie tracking shell +- Cross-platform mobile app (iOS/Android) +- Spring Boot backend API +- Python OCR worker for timetable processing + +### Features +- **Timetable Management**: Import and manage weekly timetables +- **Myspace**: Personal vault for screenshots, notes, and links +- **Hangout**: Social features for student interaction +- **Calorie Tracking**: Basic calorie monitoring +- **Cross-Platform**: Single codebase for iOS and Android + +### Technical +- Expo React Native frontend +- Spring Boot backend with PostgreSQL +- Python ML worker for OCR processing +- Neo4j for graph-based data relationships +- Vector search capabilities for content discovery \ No newline at end of file