diff --git a/.github/workflows/generate-content.yml b/.github/workflows/generate-content.yml index 7b01d25..22b08e9 100644 --- a/.github/workflows/generate-content.yml +++ b/.github/workflows/generate-content.yml @@ -20,6 +20,10 @@ on: push: branches: - main + paths-ignore: + - 'src/content/glossary/**' + - 'data/terms.csv' + - 'logs/**' jobs: generate: @@ -43,7 +47,7 @@ jobs: - name: Install dependencies run: | npm install - pip install requests + pip install -r requirements.txt - name: Generate content env: @@ -114,8 +118,8 @@ jobs: exit 0 fi - git add content/glossary/ data/terms.csv logs/ - ARTICLES_COUNT=$(ls -1 content/glossary/*.md 2>/dev/null | wc -l) + git add src/content/glossary/ data/terms.csv logs/ + ARTICLES_COUNT=$(ls -1 src/content/glossary/*.md 2>/dev/null | wc -l) TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') git commit -m "chore: auto-generate glossary content diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml index af21613..e223ddf 100644 --- a/.github/workflows/maintenance.yml +++ b/.github/workflows/maintenance.yml @@ -62,7 +62,7 @@ jobs: # Check all markdown files broken_links = [] - glossary_dir = Path('content/glossary') + glossary_dir = Path('src/content/glossary') if glossary_dir.exists(): for md_file in glossary_dir.glob('*.md'): @@ -155,7 +155,7 @@ jobs: cluster_count[row['cluster']] += 1 # Count generated files - glossary_dir = Path('content/glossary') + glossary_dir = Path('src/content/glossary') generated_count = len(list(glossary_dir.glob('*.md'))) if glossary_dir.exists() else 0 print("**Terms by Status:**") diff --git a/.gitignore b/.gitignore index fd10cfd..b669145 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ dist/ .DS_Store *.log logs/ -content/glossary/*.md .astro/ .vscode/ .idea/ diff --git a/astro.config.mjs b/astro.config.mjs index 204adf4..5309ab2 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -4,7 +4,7 @@ import tailwind from '@astrojs/tailwind'; export default defineConfig({ integrations: [tailwind()], output: 'static', - site: 'https://ai-glossary.example.com', + site: process.env.SITE_URL || 'https://ai-glossary.pages.dev', vite: { ssr: { external: ['svgo'] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6bba895 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +openai>=1.0.0 +google-generativeai>=0.3.0 +requests>=2.28.0 diff --git a/scripts/api_client.py b/scripts/api_client.py index c1d393a..96b34ee 100644 --- a/scripts/api_client.py +++ b/scripts/api_client.py @@ -3,7 +3,7 @@ import logging from typing import Optional, List from datetime import datetime -import anthropic +from openai import OpenAI import google.generativeai as genai logging.basicConfig(level=logging.INFO) @@ -107,16 +107,29 @@ def generate( else: raise ValueError(f"Unknown provider: {self.provider}") + def _is_rate_limit_error(self, error_str: str) -> bool: + """Check if error is a rate limit / quota issue.""" + rate_limit_indicators = [ + "rate_limit", + "rate limit", + "quota", + "too many requests", + "429", + "tokens per minute", + "requests per minute", + "requests per day", + "resource_exhausted", + ] + return any(indicator in error_str for indicator in rate_limit_indicators) + def _generate_groq(self, prompt: str, term: str, model: str, max_retries: int) -> Optional[str]: """Generate using Groq API with key rotation on rate limit.""" if not self.groq_keys: - # Fallback to single key from env - api_key = os.getenv("GROQ_API_KEY") - if not api_key: - raise ValueError("GROQ_API_KEY not set") - return self._try_groq_generation(api_key, prompt, term, model, max_retries, 0) + raise ValueError("GROQ_API_KEY not set or empty") + + # Save starting index to detect full loop + start_index = self.current_groq_index - # Try with multiple keys on rate limit for key_attempt in range(len(self.groq_keys)): api_key, key_index = self._get_next_groq_key() @@ -124,55 +137,49 @@ def _generate_groq(self, prompt: str, term: str, model: str, max_retries: int) - if result: return result - # If failed, rotate to next key - if key_attempt < len(self.groq_keys) - 1: - self._rotate_groq_key("exhausted") - logger.info(f"Trying next Groq API key...") + # Always rotate after failure so next term starts on fresh key + self._rotate_groq_key("exhausted") + logger.info(f"Trying next Groq API key...") - logger.error(f"All Groq API keys exhausted for {term}") + logger.error(f"All {len(self.groq_keys)} Groq API keys exhausted for '{term}'") return None def _try_groq_generation( self, api_key: str, prompt: str, term: str, model: str, max_retries: int, key_index: int ) -> Optional[str]: - """Try generation with single Groq API key.""" - client = anthropic.Anthropic(api_key=api_key, base_url="https://api.groq.com/openai/v1") + """Try generation with single Groq API key (OpenAI-compatible).""" + client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1") for attempt in range(max_retries): try: - response = client.messages.create( + response = client.chat.completions.create( model=model, max_tokens=2000, messages=[{"role": "user", "content": prompt}], ) - content = response.content[0].text - tokens = response.usage.output_tokens + response.usage.input_tokens + content = response.choices[0].message.content + tokens = (response.usage.completion_tokens or 0) + (response.usage.prompt_tokens or 0) self._log_usage(tokens, model, term, key_index) return content except Exception as e: error_str = str(e).lower() - if "rate_limit" in error_str or "quota" in error_str: + if self._is_rate_limit_error(error_str): if attempt < max_retries - 1: wait_time = 2 ** attempt - logger.warning(f"Rate limited on key {key_index}. Retrying in {wait_time}s...") + logger.warning(f"Rate limited on key {key_index}, attempt {attempt+1}/{max_retries}. Retrying in {wait_time}s...") time.sleep(wait_time) else: - logger.warning(f"Rate limit exceeded on key {key_index}, will try next key") + logger.warning(f"Rate limit exceeded on key {key_index} after {max_retries} retries, will try next key") return None else: - logger.error(f"Generation failed on key {key_index}: {e}") + logger.error(f"Non-rate-limit error on key {key_index}: {e}") return None def _generate_gemini(self, prompt: str, term: str, model: str, max_retries: int) -> Optional[str]: """Generate using Google Gemini API with key rotation on rate limit.""" if not self.gemini_keys: - # Fallback to single key from env - api_key = os.getenv("GEMINI_API_KEY") - if not api_key: - raise ValueError("GEMINI_API_KEY not set") - return self._try_gemini_generation(api_key, prompt, term, model, max_retries, 0) + raise ValueError("GEMINI_API_KEY not set or empty") - # Try with multiple keys on rate limit for key_attempt in range(len(self.gemini_keys)): api_key, key_index = self._get_next_gemini_key() @@ -180,12 +187,11 @@ def _generate_gemini(self, prompt: str, term: str, model: str, max_retries: int) if result: return result - # If failed, rotate to next key - if key_attempt < len(self.gemini_keys) - 1: - self._rotate_gemini_key("exhausted") - logger.info(f"Trying next Gemini API key...") + # Always rotate after failure so next term starts on fresh key + self._rotate_gemini_key("exhausted") + logger.info(f"Trying next Gemini API key...") - logger.error(f"All Gemini API keys exhausted for {term}") + logger.error(f"All {len(self.gemini_keys)} Gemini API keys exhausted for '{term}'") return None def _try_gemini_generation( @@ -205,14 +211,14 @@ def _try_gemini_generation( return content except Exception as e: error_str = str(e).lower() - if "rate_limit" in error_str or "quota" in error_str or "resource_exhausted" in error_str: + if self._is_rate_limit_error(error_str): if attempt < max_retries - 1: wait_time = 2 ** attempt - logger.warning(f"Rate limited on key {key_index}. Retrying in {wait_time}s...") + logger.warning(f"Rate limited on key {key_index}, attempt {attempt+1}/{max_retries}. Retrying in {wait_time}s...") time.sleep(wait_time) else: - logger.warning(f"Rate limit exceeded on key {key_index}, will try next key") + logger.warning(f"Rate limit exceeded on key {key_index} after {max_retries} retries, will try next key") return None else: - logger.error(f"Generation failed on key {key_index}: {e}") + logger.error(f"Non-rate-limit error on key {key_index}: {e}") return None diff --git a/scripts/generate.py b/scripts/generate.py index bd0c22d..3722a73 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -2,6 +2,7 @@ import os import sys import csv +import random import argparse import logging from datetime import datetime @@ -61,7 +62,7 @@ def _get_pending_terms(self, terms: list, priority: int = None) -> list: def _slug_exists(self, slug: str) -> bool: """Check if markdown file already exists for slug.""" - return os.path.exists(f"content/glossary/{slug}.md") + return os.path.exists(f"src/content/glossary/{slug}.md") def _build_prompt(self, term: str, slug: str, cluster: str) -> str: """Build AI prompt from template.""" @@ -80,8 +81,8 @@ def _extract_keywords(self, term: str) -> list: keywords.extend(words[:4]) return keywords[:5] - def generate_batch(self, batch_size: int = 20, priority: int = None, dry_run: bool = False): - """Generate next N pending terms.""" + def generate_batch(self, batch_size: int = 20, priority: int = None, dry_run: bool = False, random_pick: bool = False): + """Generate next N pending terms. If random_pick=True, shuffle before selecting.""" terms = self._read_terms() pending = self._get_pending_terms(terms, priority) @@ -90,6 +91,11 @@ def generate_batch(self, batch_size: int = 20, priority: int = None, dry_run: bo self.notifier.send_message("⚠️ No pending terms to generate") return + # Shuffle for random selection + if random_pick: + random.shuffle(pending) + logger.info(f"Randomized pending terms order") + logger.info(f"Found {len(pending)} pending terms. Generating {min(batch_size, len(pending))}...") self.notifier.notify_generation_start(batch_size, priority) @@ -235,7 +241,7 @@ def main(): if args.random < 10 or args.random > 50: logger.error(f"Random batch size must be between 10-50, got {args.random}") sys.exit(1) - generator.generate_batch(batch_size=args.random, priority=args.priority, dry_run=args.dry_run) + generator.generate_batch(batch_size=args.random, priority=args.priority, dry_run=args.dry_run, random_pick=True) else: generator.generate_batch(batch_size=batch_size, priority=args.priority, dry_run=args.dry_run) diff --git a/scripts/internal_links.py b/scripts/internal_links.py index 8582364..5b1db0b 100644 --- a/scripts/internal_links.py +++ b/scripts/internal_links.py @@ -11,7 +11,7 @@ class InternalLinker: def __init__(self): - self.glossary_dir = "content/glossary" + self.glossary_dir = "src/content/glossary" self.terms_csv = "data/terms.csv" self.max_links_per_article = 3 diff --git a/scripts/markdown_writer.py b/scripts/markdown_writer.py index a5754dc..a9aa713 100644 --- a/scripts/markdown_writer.py +++ b/scripts/markdown_writer.py @@ -27,7 +27,7 @@ def _log_write(self, filepath: str, status: str): def write(self, slug: str, content: str) -> bool: """Write markdown file with safety checks. Returns True if written, False if skipped.""" - filepath = f"content/glossary/{slug}.md" + filepath = f"src/content/glossary/{slug}.md" # Ensure directory exists os.makedirs(os.path.dirname(filepath), exist_ok=True) diff --git a/src/content.config.ts b/src/content.config.ts new file mode 100644 index 0000000..69ade53 --- /dev/null +++ b/src/content.config.ts @@ -0,0 +1,17 @@ +import { defineCollection, z } from 'astro:content'; +import { glob } from 'astro/loaders'; + +const glossary = defineCollection({ + loader: glob({ pattern: '**/*.md', base: './src/content/glossary' }), + schema: z.object({ + title: z.string(), + slug: z.string(), + description: z.string(), + keywords: z.array(z.string()).default([]), + cluster: z.string().default(''), + related_terms: z.array(z.string()).default([]), + created_at: z.string().optional(), + }), +}); + +export const collections = { glossary }; diff --git a/src/content/glossary/.gitkeep b/src/content/glossary/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/layouts/Layout.astro b/src/layouts/Layout.astro index 72727fd..59626ec 100644 --- a/src/layouts/Layout.astro +++ b/src/layouts/Layout.astro @@ -14,11 +14,11 @@ const { title, description } = Astro.props; {title} — AI Glossary for Beginners {description && } - + - + + +
diff --git a/src/pages/glossary/[slug].astro b/src/pages/glossary/[slug].astro index a0265c8..4da4398 100644 --- a/src/pages/glossary/[slug].astro +++ b/src/pages/glossary/[slug].astro @@ -1,46 +1,20 @@ --- import Layout from '../../layouts/Layout.astro'; -import { getCollection } from 'astro:content'; +import { getCollection, render } from 'astro:content'; export async function getStaticPaths() { - const glossaryFiles = import.meta.glob('../../content/glossary/*.md', { eager: true }); - - return Object.entries(glossaryFiles).map(([path, file]) => { - const slug = path.split('/').pop()?.replace('.md', ''); - return { - params: { slug }, - props: { file } - }; - }); -} + const glossaryEntries = await getCollection('glossary'); -interface Props { - file: any; + return glossaryEntries.map((entry) => ({ + params: { slug: entry.id }, + props: { entry }, + })); } -const { file } = Astro.props; -const { slug } = Astro.params; - -// Parse frontmatter and content -const content = file.default?.toString() || ''; -const parts = content.split('---'); -const frontmatterText = parts[1] || ''; -const body = parts.slice(2).join('---').trim(); - -// Parse frontmatter -const frontmatter: Record = {}; -frontmatterText.split('\n').forEach(line => { - if (line.includes(':')) { - const [key, ...valueParts] = line.split(':'); - frontmatter[key.trim()] = valueParts.join(':').trim(); - } -}); +const { entry } = Astro.props; +const { Content } = await render(entry); -const title = frontmatter.title || 'Untitled'; -const description = frontmatter.description || ''; -const keywords = frontmatter.keywords || []; -const cluster = frontmatter.cluster || ''; -const relatedTerms = frontmatter.related_terms || []; +const { title, description, keywords, cluster, related_terms, created_at } = entry.data; --- @@ -57,29 +31,31 @@ const relatedTerms = frontmatter.related_terms || [];

{title}

{description}

-
- {keywords.map((keyword: string) => ( - - {keyword} - - ))} -
+ {keywords && keywords.length > 0 && ( +
+ {keywords.map((keyword: string) => ( + + {keyword} + + ))} +
+ )}

By AI Glossary Team

-

Published: {frontmatter.created_at}

+ {created_at &&

Published: {created_at}

}
- +
- {relatedTerms && relatedTerms.length > 0 && ( + {related_terms && related_terms.length > 0 && (

Related Terms

- {relatedTerms.map((term: string) => ( + {related_terms.map((term: string) => (

{term}

 diff --git a/src/pages/index.astro b/src/pages/index.astro index 202c7de..31dfa95 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -1,5 +1,6 @@ --- import Layout from '../layouts/Layout.astro'; +import { getCollection } from 'astro:content'; const title = "AI Glossary for Beginners"; const description = "AI explained simply for normal humans. Learn AI concepts in plain language."; @@ -9,14 +10,14 @@ const clusters = [ { id: 'llms', name: 'LLMs & Models', icon: '🤖' }, { id: 'prompt-engineering', name: 'Prompt Engineering', icon: '✍️' }, { id: 'ai-tools', name: 'AI Tools & Apps', icon: '🛠️' }, + { id: 'ai-agents', name: 'AI Agents', icon: '🕵️' }, + { id: 'ai-infrastructure', name: 'AI Infrastructure', icon: '⚙️' }, ]; -const featuredTerms = [ - { slug: 'what-is-an-llm', title: 'What is an LLM?' }, - { slug: 'what-is-prompt-engineering', title: 'What is Prompt Engineering?' }, - { slug: 'what-is-rag', title: 'What is RAG?' }, - { slug: 'what-is-an-ai-agent', title: 'What is an AI Agent?' }, -]; +// Get actual published articles +const allArticles = await getCollection('glossary'); +const latestArticles = allArticles.slice(0, 4); +const totalCount = allArticles.length; --- @@ -29,44 +30,60 @@ const featuredTerms = [ Search Terms - - Browse A–Z - + {totalCount > 0 && ( + + {totalCount} terms explained + + )}
-
-

Start Here

-
- {featuredTerms.map(term => ( - -

{term.title}

- - ))} -
-
+ {latestArticles.length > 0 && ( +
+

Latest Articles

+
+ {latestArticles.map(article => ( + +

{article.data.title}

+

{article.data.description}

+ + ))} +
+
+ )}

Topics

-
+
{clusters.map(cluster => ( - +
{cluster.icon}

{cluster.name}

-

Explore this topic

- +

+ {allArticles.filter(a => a.data.cluster === cluster.id).length} articles +

+
))}
-
-

Browse Alphabetically

-
- {Array.from('ABCDEFGHIJKLMNOPQRSTUVWXYZ').map(letter => ( - - {letter} - - ))} -
-
+ {totalCount > 0 && ( +
+

Browse Alphabetically

+
+ {Array.from('ABCDEFGHIJKLMNOPQRSTUVWXYZ').map(letter => { + const count = allArticles.filter(a => a.data.title.toUpperCase().startsWith(letter)).length; + return count > 0 ? ( + + {letter} ({count}) + + ) : ( + + {letter} + + ); + })} +
+
+ )}