[deploy] gemini API 키값 RR 분산 적용 배포

README.md

@@ -77,7 +77,8 @@ pip install -r requirements.txt
 # LOCAL_DB_HOST, LOCAL_DB_PORT, LOCAL_DB_NAME, LOCAL_DB_USER, LOCAL_DB_PASSWORD
 # REDIS_LOCAL_HOST, REDIS_LOCAL_PORT
 # JWT_SECRET, JWT_ALGORITHM
-# GEMINI_API_KEY
+# GEMINI_API_KEY          (단일 키, 하위 호환)
+# GEMINI_API_KEYS=key1,key2,key3   (다중 키 round-robin, 설정 시 KEYS 우선)
 # REDIS_LOCAL_HOST, REDIS_LOCAL_PORT  (일일 사용 제한; local)
 # AI_PIN_GENERATION_* / ISSUE_PIN_CREATE_* / ISSUE_PIN_EDIT_*  (rate limit, docs/issue-pin-create-and-reliability.md §9)
 # AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_BUCKET (이미지 업로드 시)

app/core/config.py

@@ -94,6 +94,11 @@ class Settings(BaseSettings):
 
     # Gemini/Vector DB
     gemini_api_key: SecretStr | None = Field(default=None, alias="GEMINI_API_KEY")
+    gemini_api_keys: str = Field(
+        default="",
+        alias="GEMINI_API_KEYS",
+        description="콤마 구분 다중 키 (설정 시 GEMINI_API_KEY보다 우선)",
+    )
     # 이슈 핀 신뢰도 VLM. 3.1-pro는 503·지연이 잦아 기본은 flash 계열.
     gemini_vlm_model: str = Field(
         default="gemini-2.5-flash",
@@ -669,6 +674,17 @@ def async_database_url(self) -> str:
             db_password=conn.password,
         )
 
+    def resolved_gemini_api_keys(self) -> tuple[str, ...]:
+        from app.services.internal.ai.gemini_retry import parse_gemini_model_list
+
+        if self.gemini_api_keys.strip():
+            raw = list(parse_gemini_model_list(self.gemini_api_keys))
+        elif self.gemini_api_key is not None:
+            raw = [self.gemini_api_key.get_secret_value()]
+        else:
+            raw = []
+        return tuple(dict.fromkeys(k for k in raw if k))
+
 
 @lru_cache
 def get_settings() -> Settings:

app/core/deps.py

@@ -48,6 +48,7 @@
     build_vlm_service,
     require_gemini_api_key,
 )
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool, get_gemini_key_pool
 from app.services.internal.complaint_wiring import build_complaint_email_service
 from app.services.internal.geo.ImageExifLocationResolveService import ImageExifLocationResolveService
 from app.services.internal.geo.ImageMultipartGeoService import ImageMultipartGeoService
@@ -148,6 +149,14 @@ def get_vector_store_service(request: Request) -> VectorStoreService:
 VectorStoreServiceDep = Annotated[VectorStoreService, Depends(get_vector_store_service)]
 
 
+def get_gemini_key_pool_dep(request: Request) -> GeminiKeyPool | None:
+    pool = getattr(request.app.state, "gemini_key_pool", None)
+    return pool or get_gemini_key_pool()
+
+
+GeminiKeyPoolDep = Annotated[GeminiKeyPool | None, Depends(get_gemini_key_pool_dep)]
+
+
 def get_pin_repo(session: DbSessionDep) -> PinRepo:
     return PinRepo(session)
 

app/main.py

@@ -21,6 +21,7 @@
 from app.services.internal.ContestPinSchedulerService import ContestPinSchedulerService
 from app.services.internal.PolicyPinSchedulerService import PolicyPinSchedulerService
 from app.services.internal.complaint_wiring import build_complaint_email_service
+from app.services.internal.ai.gemini_key_pool import init_gemini_key_pool
 from app.services.VectorStoreService import VectorStoreService
 from app.services.vector_domains import build_hnsw_kwargs, build_vector_domain_configs
 from app.schemas.IssueDTO import (
@@ -89,14 +90,23 @@ async def lifespan(app: FastAPI):
                 exc,
             )
 
-    gemini_api_key_secret = settings.gemini_api_key
-    if gemini_api_key_secret is None:
-        logger.warning("GEMINI_API_KEY is not configured. VectorStoreService initialization skipped.")
+    gemini_api_keys = settings.resolved_gemini_api_keys()
+    gemini_key_pool = init_gemini_key_pool()
+    if gemini_key_pool is not None:
+        app.state.gemini_key_pool = gemini_key_pool
+        logger.info(
+            "Gemini key pool initialized: %d key(s), rotation=%s",
+            gemini_key_pool.size,
+            "on" if gemini_key_pool.enabled else "off",
+        )
     else:
+        logger.warning("GEMINI_API_KEY is not configured. VectorStoreService initialization skipped.")
+
+    if gemini_api_keys:
         try:
             domain_configs = build_vector_domain_configs(settings)
             app.state.vector_store_service = VectorStoreService(
-                api_key=gemini_api_key_secret.get_secret_value(),
+                api_key=gemini_api_keys[0],
                 table_name=settings.vector_table_name,
                 default_embedding_model=settings.gemini_embedding_model,
                 default_embed_dim=settings.vector_embed_dim,
@@ -105,6 +115,7 @@ async def lifespan(app: FastAPI):
                 text_search_config=settings.vector_text_search_config,
                 embedding_batch_size_override=settings.gemini_embedding_batch_size,
                 hnsw_kwargs=build_hnsw_kwargs(settings),
+                key_pool=gemini_key_pool,
             )
             if settings.vector_dim_check:
                 dimension_checks = (
@@ -149,12 +160,12 @@ async def lifespan(app: FastAPI):
 
     complaint_scheduler = None
     if (
-        gemini_api_key_secret is not None
+        gemini_api_keys
         and getattr(app.state, "vector_store_service", None) is not None
     ):
         try:
             complaint_email_service = build_complaint_email_service(
-                api_key=gemini_api_key_secret.get_secret_value(),
+                api_key=gemini_api_keys[0],
                 vector_store_service=app.state.vector_store_service,
             )
             complaint_scheduler = ComplaintPetitionSchedulerService(
@@ -167,7 +178,7 @@ async def lifespan(app: FastAPI):
             logger.warning("Complaint scheduler initialization failed: %s", exc)
 
     policy_pin_scheduler = None
-    if settings.policy_news_service_key is not None and gemini_api_key_secret is not None:
+    if settings.policy_news_service_key is not None and gemini_api_keys:
         try:
             policy_pin_scheduler = PolicyPinSchedulerService(s3_util=app.state.s3_util)
             policy_pin_scheduler.start()
@@ -176,7 +187,7 @@ async def lifespan(app: FastAPI):
             logger.warning("Policy pin scheduler initialization failed: %s", exc)
 
     contest_pin_scheduler = None
-    if gemini_api_key_secret is not None:
+    if gemini_api_keys:
         try:
             contest_pin_scheduler = ContestPinSchedulerService(s3_util=app.state.s3_util)
             contest_pin_scheduler.start()

app/services/RagRerankService.py

@@ -8,6 +8,9 @@
 import numpy as np
 from llama_index.embeddings.google_genai import GoogleGenAIEmbedding
 
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
+from app.services.internal.ai.rotating_gemini_embedding import RotatingGoogleGenAIEmbedding
+
 logger = logging.getLogger(__name__)
 
 
@@ -26,16 +29,26 @@ def __init__(
         embedding_model: str,
         embed_dim: int,
         embedding_batch_size: int | None = None,
+        key_pool: GeminiKeyPool | None = None,
     ) -> None:
         batch_size = embedding_batch_size
         if batch_size is None:
             batch_size = 1 if embedding_model.strip().endswith("embedding-2") else 10
-        self._embed_model = GoogleGenAIEmbedding(
-            model_name=embedding_model,
-            api_key=api_key,
-            embedding_config={"output_dimensionality": embed_dim},
-            embed_batch_size=max(1, batch_size),
-        )
+        resolved_batch_size = max(1, batch_size)
+        if key_pool is not None and key_pool.enabled:
+            self._embed_model = RotatingGoogleGenAIEmbedding(
+                key_pool=key_pool,
+                model_name=embedding_model,
+                embed_dim=embed_dim,
+                embed_batch_size=resolved_batch_size,
+            )
+        else:
+            self._embed_model = GoogleGenAIEmbedding(
+                model_name=embedding_model,
+                api_key=api_key,
+                embedding_config={"output_dimensionality": embed_dim},
+                embed_batch_size=resolved_batch_size,
+            )
 
     async def rerank(
         self,

app/services/VectorStoreService.py

@@ -15,6 +15,8 @@
 
 from app.core.database import async_database_url as default_async_database_url
 from app.core.database import sync_database_url as default_sync_database_url
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
+from app.services.internal.ai.rotating_gemini_embedding import RotatingGoogleGenAIEmbedding
 from app.services.vector_domains import DomainVectorConfig, VectorDomain
 
 logger = logging.getLogger(__name__)
@@ -47,6 +49,7 @@ def __init__(
         text_search_config: str = "simple",
         embedding_batch_size_override: int | None = None,
         hnsw_kwargs: dict[str, Any] | None = None,
+        key_pool: GeminiKeyPool | None = None,
     ) -> None:
         self._sync_database_url = sync_database_url
         self._async_database_url = async_database_url
@@ -56,6 +59,7 @@ def __init__(
         self._hybrid_search = hybrid_search
         self._text_search_config = text_search_config
         self._api_key = api_key
+        self._key_pool = key_pool
         self._bundles: dict[str, _VectorIndexBundle] = {}
         self._domain_configs = domain_configs or {}
         self._embed_models: dict[tuple[str, int], BaseEmbedding] = {}
@@ -106,12 +110,20 @@ def _get_or_create_embed_model(self, model_name: str, embed_dim: int) -> BaseEmb
             embed_batch_size = max(1, int(self._embedding_batch_size_override))
         else:
             embed_batch_size = 1 if model_name.strip().endswith("embedding-2") else 10
-        embed_model = GoogleGenAIEmbedding(
-            model_name=model_name,
-            api_key=self._api_key,
-            embedding_config={"output_dimensionality": embed_dim},
-            embed_batch_size=embed_batch_size,
-        )
+        if self._key_pool is not None and self._key_pool.enabled:
+            embed_model = RotatingGoogleGenAIEmbedding(
+                key_pool=self._key_pool,
+                model_name=model_name,
+                embed_dim=embed_dim,
+                embed_batch_size=embed_batch_size,
+            )
+        else:
+            embed_model = GoogleGenAIEmbedding(
+                model_name=model_name,
+                api_key=self._api_key,
+                embedding_config={"output_dimensionality": embed_dim},
+                embed_batch_size=embed_batch_size,
+            )
         self._embed_models[model_key] = embed_model
         return embed_model
 

app/services/internal/ai/ComplaintEmailLLMService.py

@@ -15,6 +15,7 @@
     ComplaintEmailOpinionRenderer,
     OpinionAttachmentImage,
 )
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
 from app.services.internal.ai.gemini_retry import generate_content_with_retry
 from app.services.prompts.complaint_email_opinion import complaint_opinion_prompt
 
@@ -25,6 +26,7 @@
 class ComplaintEmailLLMService:
     api_key: str
     model_name: str = "gemini-2.5-flash"
+    key_pool: GeminiKeyPool | None = None
     client: genai.Client = field(init=False, repr=False)
     _opinion_renderer: ComplaintEmailOpinionRenderer = field(
         default_factory=ComplaintEmailOpinionRenderer,
@@ -75,6 +77,7 @@ async def _generate_json_text(self, prompt: str) -> str:
                 contents=text,
                 config=config,
                 log_prefix="ComplaintEmailLLM",
+                key_pool=self.key_pool,
             )
         except genai_errors.APIError as exc:
             raise BusinessException(

app/services/internal/ai/ComplaintEmailVLMService.py

@@ -17,6 +17,7 @@
     ComplaintEmailVlmInput,
 )
 from app.schemas.IssueDTO import ImageWithLocation
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
 from app.services.internal.ai.gemini_retry import generate_content_with_retry
 from app.services.internal.ai.VLMService import resolve_upload_image_mime
 from app.services.prompts.complaint_email_vlm import (
@@ -111,12 +112,14 @@ def __init__(
         catalog: type[ComplaintEmailVlmCatalog] = ComplaintEmailVlmCatalog,
         prompt_builder: ComplaintEmailVlmPromptBuilder | None = None,
         result_processor: ComplaintEmailVlmResultProcessor | None = None,
+        key_pool: GeminiKeyPool | None = None,
     ) -> None:
         self._api_key = api_key
         self._model = model
         self._catalog = catalog
         self._prompt_builder = prompt_builder or ComplaintEmailVlmPromptBuilder(catalog)
         self._processor = result_processor or ComplaintEmailVlmResultProcessor(catalog)
+        self._key_pool = key_pool
         self._client: genai.Client | None = None
         if api_key:
             self._client = genai.Client(api_key=api_key)
@@ -199,6 +202,7 @@ async def _call_vlm(self, prompt: str, images: list[ImageWithLocation]) -> str:
                 contents=parts,
                 config=config,
                 log_prefix="ComplaintEmailVLM",
+                key_pool=self._key_pool,
             )
         except genai_errors.APIError as exc:
             raise BusinessException(

app/services/internal/ai/IssuePinLLMService.py

@@ -11,6 +11,7 @@
 
 from app.core.codes import ErrorCode
 from app.core.exceptions import BusinessException, raise_business_exception
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
 from app.services.internal.ai.gemini_retry import generate_content_with_retry
 from app.services.prompts.issue_pin import ISSUE_PIN_OUTPUT_SCHEMA
 
@@ -23,6 +24,7 @@ class IssuePinLLMService:
 
     api_key: str
     model_name: str = "gemini-2.5-flash"
+    key_pool: GeminiKeyPool | None = None
     client: genai.Client = field(init=False, repr=False)
     fallback_models: tuple[str, ...] = ("gemini-2.5-flash-lite", "gemini-2.5-flash")
 
@@ -42,6 +44,7 @@ async def _generate_with_retry(
             contents=contents,
             config=config,
             log_prefix="Pin",
+            key_pool=self.key_pool,
         )
 
     async def generate_pin_copy(self, *, prompt: str) -> dict[str, str]:

app/services/internal/ai/IssueRagPlannerService.py

@@ -9,6 +9,7 @@
 from google import genai
 from google.genai import types
 
+from app.services.internal.ai.gemini_key_pool import GeminiKeyPool
 from app.services.internal.ai.gemini_retry import generate_content_with_retry
 
 logger = logging.getLogger(__name__)
@@ -74,6 +75,7 @@ def _normalize_string_list(value: object) -> list[str]:
 class IssueRagPlannerService:
     api_key: str
     model_name: str = "gemini-2.5-flash-lite"
+    key_pool: GeminiKeyPool | None = None
     client: genai.Client = field(init=False, repr=False)
     fallback_models: tuple[str, ...] = ("gemini-2.5-flash",)
 
@@ -95,6 +97,7 @@ async def _generate_with_retry(
             config=config,
             log_prefix="Planner",
             log_context=log_context,
+            key_pool=self.key_pool,
         )
 
     async def rewrite_queries(