diff --git a/.github/workflows/prod_deploy.yml b/.github/workflows/prod_deploy.yml new file mode 100644 index 0000000..b5d2cf9 --- /dev/null +++ b/.github/workflows/prod_deploy.yml @@ -0,0 +1,64 @@ +name: Issueissyu Prod CI/CD + +on: + pull_request: + types: + - closed + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || + (github.event.pull_request.merged == true && (github.event.pull_request.base.ref == 'prod' || github.event.pull_request.base.ref == 'test/CICD')) + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create .env file from secret + env: + DEV_ENV: ${{ secrets.PROD_ENV }} + run: | + printf '%s' "$DEV_ENV" > .env + chmod 600 .env + + - name: Get current time + uses: josStorer/get-current-time@v2 + id: current-time + with: + format: YYYY-MM-DDTHH:mm:ss + utcOffset: "+09:00" + + - name: Generate deployment package + run: | + mkdir -p deploy + rsync -a \ + --exclude='.git' \ + --exclude='deploy' \ + --exclude='__pycache__' \ + --exclude='.venv' \ + --exclude='venv' \ + --exclude='.idea' \ + --exclude='.cursor' \ + --exclude='.github' \ + --exclude='*.pyc' \ + ./ deploy/ + (cd deploy && zip -r deploy.zip .) + + - name: Deploy to Elastic Beanstalk + uses: einaregilsson/beanstalk-deploy@v22 + with: + aws_access_key: ${{ secrets.AWS_PROD_ACCESS_KEY }} + aws_secret_key: ${{ secrets.AWS_PROD_SECRET_KEY }} + application_name: 'issueissyu-ai-prod' + environment_name: 'issueissyu-ai-env' + region: 'ap-northeast-2' + version_label: github-action-${{ steps.current-time.outputs.formattedTime }} + version_description: ${{ github.ref_name }}@${{ github.sha }} + deployment_package: deploy/deploy.zip + wait_for_deployment: false diff --git a/app/core/config.py b/app/core/config.py index 6f3ce80..d89dbd8 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -55,6 +55,8 @@ class Settings(BaseSettings): aws_secret_key: SecretStr | None = Field(default=None, alias="AWS_SECRET_KEY") aws_region: str = Field(default="us-east-1", alias="AWS_REGION") aws_bucket_name: str | None = Field(default=None, alias="AWS_BUCKET") + cdn_enabled: bool = Field(default=False, alias="CDN_ENABLED") + cdn_base_url: str | None = Field(default=None, alias="CDN_BASE_URL") # SMTP (민원 이메일 실제 송신) smtp_host: str | None = Field(default=None, alias="SMTP_HOST") @@ -158,6 +160,17 @@ class Settings(BaseSettings): default="simple", alias="VECTOR_TEXT_SEARCH_CONFIG", ) + vector_hnsw_m: int = Field(default=16, ge=1, alias="VECTOR_HNSW_M") + vector_hnsw_ef_construction: int = Field( + default=64, + ge=1, + alias="VECTOR_HNSW_EF_CONSTRUCTION", + ) + vector_hnsw_ef_search: int = Field(default=40, ge=1, alias="VECTOR_HNSW_EF_SEARCH") + vector_hnsw_dist_method: str = Field( + default="vector_cosine_ops", + alias="VECTOR_HNSW_DIST_METHOD", + ) # 문화체육관광부 정책브리핑 정책뉴스 OpenAPI (공공데이터포털) policy_news_service_key: SecretStr | None = Field( @@ -398,6 +411,20 @@ def _empty_string_gemini_embed_batch(cls, value: object) -> object: return None return value + @field_validator("cdn_base_url", mode="before") + @classmethod + def _empty_cdn_base_url_to_none(cls, value: object) -> object | None: + if isinstance(value, str) and value.strip() == "": + return None + return value + + @field_validator("vector_hnsw_dist_method", mode="before") + @classmethod + def _empty_vector_hnsw_dist_method(cls, value: object) -> object: + if isinstance(value, str) and value.strip() == "": + return "vector_cosine_ops" + return value + @field_validator( "local_db_port", "aws_db_port", diff --git a/app/main.py b/app/main.py index 5060cd0..51dbfd7 100644 --- a/app/main.py +++ b/app/main.py @@ -22,7 +22,7 @@ from app.services.internal.PolicyPinSchedulerService import PolicyPinSchedulerService from app.services.internal.complaint_wiring import build_complaint_email_service from app.services.VectorStoreService import VectorStoreService -from app.services.vector_domains import build_vector_domain_configs +from app.services.vector_domains import build_hnsw_kwargs, build_vector_domain_configs from app.schemas.IssueDTO import ( CreateIssuePinMultipartRequest, PinImageIsMainItem, @@ -104,6 +104,7 @@ async def lifespan(app: FastAPI): hybrid_search=settings.vector_hybrid_search, text_search_config=settings.vector_text_search_config, embedding_batch_size_override=settings.gemini_embedding_batch_size, + hnsw_kwargs=build_hnsw_kwargs(settings), ) if settings.vector_dim_check: dimension_checks = ( diff --git a/app/routes/TestRoute.py b/app/routes/TestRoute.py index 448501e..caac245 100644 --- a/app/routes/TestRoute.py +++ b/app/routes/TestRoute.py @@ -1,12 +1,18 @@ -from fastapi import APIRouter, File, UploadFile +import io +from pathlib import Path +from fastapi import APIRouter, File, Query, UploadFile +from fastapi.responses import StreamingResponse from app.core.deps import CurrentUserIdDep, S3UtilDep, UserServiceDep from app.core.codes import ErrorCode, SuccessCode -from app.core.exceptions import raise_business_exception +from app.core.exceptions import raise_business_exception, raise_file_exception from app.core.responses import success_response + router = APIRouter(prefix="/test", tags=["test"]) +_DEFAULT_S3_TEST_PREFIX = "test-files" + @router.post("/s3/image-upload") async def upload_test_image_to_s3( @@ -17,6 +23,61 @@ async def upload_test_image_to_s3( return success_response(result=uploaded, success_code=SuccessCode.CREATED) +@router.post("/s3/upload") +async def upload_test_file_to_s3( + s3_util: S3UtilDep, + file: UploadFile = File(...), + prefix: str = Query(default=_DEFAULT_S3_TEST_PREFIX, description="S3 object key prefix"), +): + """이미지·PDF 등 임의 파일 업로드 테스트. 응답의 key로 다운로드 API를 호출할 수 있습니다.""" + raw = await file.read() + if not raw: + raise_file_exception( + ErrorCode.FILE_UPLOAD_ERROR, + detail="업로드할 파일이 비어 있습니다.", + ) + + filename = (file.filename or "upload.bin").strip() or "upload.bin" + content_type = (file.content_type or "application/octet-stream").split(";")[0].strip() + normalized_prefix = prefix.strip("/") or _DEFAULT_S3_TEST_PREFIX + + uploaded = await s3_util.upload_binary( + raw, + filename=filename, + content_type=content_type, + prefix=normalized_prefix, + ) + return success_response(result=uploaded, success_code=SuccessCode.CREATED) + + +@router.get("/s3/download") +async def download_test_file_from_s3( + s3_util: S3UtilDep, + key: str = Query(..., min_length=1, description="S3 object key (업로드 응답의 key)"), +): + """S3 object key로 파일 다운로드 테스트.""" + data, content_type = await s3_util.download_binary(key) + filename = Path(key).name or "download.bin" + return StreamingResponse( + io.BytesIO(data), + media_type=content_type or "application/octet-stream", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@router.delete("/s3/object") +async def delete_test_file_from_s3( + s3_util: S3UtilDep, + key: str = Query(..., min_length=1, description="삭제할 S3 object key"), +): + """업로드 테스트 후 S3 객체 삭제.""" + deleted = await s3_util.delete_object(key) + return success_response( + result={"key": key, "deleted": deleted}, + success_code=SuccessCode.OK, + ) + + @router.get("/user") async def get_my_user(user_service: UserServiceDep, uid: CurrentUserIdDep): user = await user_service.get_user(uid) diff --git a/app/routes/__init__.py b/app/routes/__init__.py index 4c737cd..c5efcef 100644 --- a/app/routes/__init__.py +++ b/app/routes/__init__.py @@ -16,17 +16,17 @@ ROUTER_REGISTRY = ( {"router": user_router, "disabled_envs": set()}, {"router": issue_router, "disabled_envs": set()}, - {"router": complaint_email_router, "disabled_envs": set()}, + {"router": complaint_email_router, "disabled_envs": {"prod"}}, {"router": complaint_apply_router, "disabled_envs": set()}, - {"router": image_geo_router, "disabled_envs": set()}, + {"router": image_geo_router, "disabled_envs": {"prod"}}, {"router": test_router, "disabled_envs": {"dev", "prod"}}, {"router": vector_test_router, "disabled_envs": {"dev", "prod"}}, {"router": festival_pin_router, "disabled_envs": {"prod"}}, {"router": festival_admin_router, "disabled_envs": set()}, {"router": contest_pin_router, "disabled_envs": {"prod"}}, - {"router": contest_admin_router, "disabled_envs": {"prod"}}, + {"router": contest_admin_router, "disabled_envs": set()}, {"router": policy_pin_router, "disabled_envs": {"prod"}}, - {"router": policy_admin_router, "disabled_envs": {"prod"}}, + {"router": policy_admin_router, "disabled_envs": set()}, ) diff --git a/app/services/VectorStoreService.py b/app/services/VectorStoreService.py index 20dd973..6825670 100644 --- a/app/services/VectorStoreService.py +++ b/app/services/VectorStoreService.py @@ -19,6 +19,13 @@ logger = logging.getLogger(__name__) +DEFAULT_HNSW_KWARGS: dict[str, Any] = { + "hnsw_m": 16, + "hnsw_ef_construction": 64, + "hnsw_ef_search": 40, + "hnsw_dist_method": "vector_cosine_ops", +} + @dataclass(slots=True) class _VectorIndexBundle: table_name: str @@ -39,6 +46,7 @@ def __init__( hybrid_search: bool = True, text_search_config: str = "simple", embedding_batch_size_override: int | None = None, + hnsw_kwargs: dict[str, Any] | None = None, ) -> None: self._sync_database_url = sync_database_url self._async_database_url = async_database_url @@ -52,6 +60,8 @@ def __init__( self._domain_configs = domain_configs or {} self._embed_models: dict[tuple[str, int], BaseEmbedding] = {} self._embedding_batch_size_override = embedding_batch_size_override + self._hnsw_kwargs = hnsw_kwargs or dict(DEFAULT_HNSW_KWARGS) + logger.info("Vector HNSW config: %s", self._hnsw_kwargs) @staticmethod def _normalize_domain(domain: str) -> str: @@ -123,12 +133,7 @@ def _get_or_create_bundle( embed_dim=embed_dim, hybrid_search=self._hybrid_search, text_search_config=self._text_search_config, - hnsw_kwargs={ - "hnsw_m": 16, - "hnsw_ef_construction": 64, - "hnsw_ef_search": 40, - "hnsw_dist_method": "vector_cosine_ops", - }, + hnsw_kwargs=self._hnsw_kwargs, ) bundle = _VectorIndexBundle( table_name=resolved_table_name, diff --git a/app/services/vector_domains.py b/app/services/vector_domains.py index 45a748b..c1859d2 100644 --- a/app/services/vector_domains.py +++ b/app/services/vector_domains.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from enum import Enum +from typing import Any from app.core.config import Settings @@ -51,3 +52,12 @@ def build_vector_domain_configs(settings: Settings) -> dict[VectorDomain, Domain standalone_table=False, ), } + + +def build_hnsw_kwargs(settings: Settings) -> dict[str, Any]: + return { + "hnsw_m": settings.vector_hnsw_m, + "hnsw_ef_construction": settings.vector_hnsw_ef_construction, + "hnsw_ef_search": settings.vector_hnsw_ef_search, + "hnsw_dist_method": settings.vector_hnsw_dist_method, + } diff --git a/app/utils/S3Util.py b/app/utils/S3Util.py index d37a462..b5a880e 100644 --- a/app/utils/S3Util.py +++ b/app/utils/S3Util.py @@ -105,6 +105,9 @@ def _build_public_file_url(self, object_key: str) -> str: bucket_name = self._ensure_bucket_name() resolved_region = self.region_name or "us-east-1" encoded_key = quote(object_key.lstrip("/"), safe="/") + cdn_base_url = (settings.cdn_base_url or "").strip() + if settings.cdn_enabled and cdn_base_url: + return f"{cdn_base_url.rstrip('/')}/{encoded_key}" if resolved_region == "us-east-1": return f"https://{bucket_name}.s3.amazonaws.com/{encoded_key}" return f"https://{bucket_name}.s3.{resolved_region}.amazonaws.com/{encoded_key}" diff --git a/rag/scripts/insert_chunks_to_vector.py b/rag/scripts/insert_chunks_to_vector.py index c5cd45a..9ac0a31 100644 --- a/rag/scripts/insert_chunks_to_vector.py +++ b/rag/scripts/insert_chunks_to_vector.py @@ -11,7 +11,7 @@ # python -m rag.scripts.insert_chunks_to_vector 형태 실행 기준 from app.core.config import settings from app.services.VectorStoreService import VectorStoreService -from app.services.vector_domains import DomainVectorConfig, VectorDomain +from app.services.vector_domains import DomainVectorConfig, VectorDomain, build_hnsw_kwargs from app.utils.chunk_node_metadata import build_chunk_metadata from app.utils.chunk_text_normalize import load_skip_line_prefixes, normalize_chunk from rag.scripts.chunk_module import iter_jsonl @@ -44,6 +44,7 @@ def build_service() -> VectorStoreService: hybrid_search=settings.vector_hybrid_search, text_search_config=settings.vector_text_search_config, embedding_batch_size_override=settings.gemini_embedding_batch_size, + hnsw_kwargs=build_hnsw_kwargs(settings), ) diff --git a/tests/test_router_registry.py b/tests/test_router_registry.py new file mode 100644 index 0000000..e618442 --- /dev/null +++ b/tests/test_router_registry.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import unittest + +from app.routes import get_enabled_routers + + +def _router_tags(env: str) -> set[str]: + return {router.tags[0] for router in get_enabled_routers(env) if router.tags} + + +class RouterRegistryTest(unittest.TestCase): + def test_prod_excludes_pipeline_and_debug_routes(self) -> None: + tags = _router_tags("prod") + + self.assertNotIn("complaint-email", tags) + self.assertNotIn("geo", tags) + self.assertNotIn("festival-pins", tags) + self.assertNotIn("contest-pins", tags) + self.assertNotIn("policy-pins", tags) + self.assertNotIn("test", tags) + self.assertNotIn("vector-test", tags) + + def test_prod_includes_core_and_admin_routes(self) -> None: + tags = _router_tags("prod") + + self.assertIn("issue", tags) + self.assertIn("complaint-apply", tags) + self.assertIn("festival-admin", tags) + self.assertIn("contest-admin", tags) + self.assertIn("policy-admin", tags) + self.assertIn("users", tags) + + def test_dev_includes_complaint_email_and_geo(self) -> None: + tags = _router_tags("dev") + + self.assertIn("complaint-email", tags) + self.assertIn("geo", tags) + self.assertIn("festival-pins", tags) + self.assertIn("contest-admin", tags) + self.assertIn("policy-admin", tags) diff --git a/tests/test_s3_util.py b/tests/test_s3_util.py new file mode 100644 index 0000000..7be9b2b --- /dev/null +++ b/tests/test_s3_util.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.utils.S3Util import S3Util, settings as s3_settings + + +def _build_s3_util(*, bucket_name: str = "issueissyu-test", region_name: str = "ap-northeast-2") -> S3Util: + s3_util = S3Util.__new__(S3Util) + s3_util.bucket_name = bucket_name + s3_util.region_name = region_name + return s3_util + + +class S3UtilPublicUrlTest(unittest.TestCase): + def test_build_public_file_url_uses_cdn_when_enabled(self) -> None: + s3_util = _build_s3_util() + + with ( + patch.object(s3_settings, "cdn_enabled", True), + patch.object(s3_settings, "cdn_base_url", "https://cdn.example.com"), + ): + url = s3_util._build_public_file_url("contest-cardnews/1/slide_01.png") + + self.assertEqual(url, "https://cdn.example.com/contest-cardnews/1/slide_01.png") + + def test_build_public_file_url_trims_cdn_trailing_slash(self) -> None: + s3_util = _build_s3_util() + + with ( + patch.object(s3_settings, "cdn_enabled", True), + patch.object(s3_settings, "cdn_base_url", "https://cdn.example.com/"), + ): + url = s3_util._build_public_file_url("/contest-cardnews/1/slide_01.png") + + self.assertEqual(url, "https://cdn.example.com/contest-cardnews/1/slide_01.png") + + def test_build_public_file_url_falls_back_to_s3_when_cdn_disabled(self) -> None: + s3_util = _build_s3_util() + + with ( + patch.object(s3_settings, "cdn_enabled", False), + patch.object(s3_settings, "cdn_base_url", "https://cdn.example.com"), + ): + url = s3_util._build_public_file_url("contest-cardnews/1/slide_01.png") + + self.assertEqual( + url, + "https://issueissyu-test.s3.ap-northeast-2.amazonaws.com/contest-cardnews/1/slide_01.png", + ) + + def test_build_public_file_url_falls_back_to_s3_when_cdn_base_url_missing(self) -> None: + s3_util = _build_s3_util() + + with ( + patch.object(s3_settings, "cdn_enabled", True), + patch.object(s3_settings, "cdn_base_url", None), + ): + url = s3_util._build_public_file_url("contest-cardnews/1/slide_01.png") + + self.assertEqual( + url, + "https://issueissyu-test.s3.ap-northeast-2.amazonaws.com/contest-cardnews/1/slide_01.png", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_vector_hnsw_kwargs.py b/tests/test_vector_hnsw_kwargs.py new file mode 100644 index 0000000..7521b86 --- /dev/null +++ b/tests/test_vector_hnsw_kwargs.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from app.core.config import Settings +from app.services.VectorStoreService import DEFAULT_HNSW_KWARGS, VectorStoreService +from app.services.vector_domains import build_hnsw_kwargs + + +class BuildHnswKwargsTest(unittest.TestCase): + def test_build_hnsw_kwargs_from_settings(self) -> None: + settings = Settings( + VECTOR_HNSW_M=32, + VECTOR_HNSW_EF_CONSTRUCTION=128, + VECTOR_HNSW_EF_SEARCH=80, + VECTOR_HNSW_DIST_METHOD="vector_cosine_ops", + ) + + self.assertEqual( + build_hnsw_kwargs(settings), + { + "hnsw_m": 32, + "hnsw_ef_construction": 128, + "hnsw_ef_search": 80, + "hnsw_dist_method": "vector_cosine_ops", + }, + ) + + def test_empty_dist_method_falls_back_to_default(self) -> None: + settings = Settings(VECTOR_HNSW_DIST_METHOD="") + + self.assertEqual(settings.vector_hnsw_dist_method, "vector_cosine_ops") + + +class VectorStoreServiceHnswTest(unittest.TestCase): + def test_get_or_create_bundle_passes_hnsw_kwargs(self) -> None: + custom_kwargs = { + "hnsw_m": 24, + "hnsw_ef_construction": 96, + "hnsw_ef_search": 60, + "hnsw_dist_method": "vector_cosine_ops", + } + service = VectorStoreService( + api_key="test-key", + table_name="test_table", + default_embedding_model="gemini-embedding-2", + default_embed_dim=1536, + hnsw_kwargs=custom_kwargs, + ) + + with ( + patch("app.services.VectorStoreService.PGVectorStore") as mock_pg, + patch("app.services.VectorStoreService.VectorStoreIndex"), + patch.object(service, "_get_or_create_embed_model", return_value=MagicMock()), + ): + service._get_or_create_bundle( + resolved_table_name="complaint_chunks_complaint", + embed_model_name="gemini-embedding-2", + embed_dim=1536, + ) + + mock_pg.from_params.assert_called_once() + self.assertEqual(mock_pg.from_params.call_args.kwargs["hnsw_kwargs"], custom_kwargs) + + def test_default_hnsw_kwargs_when_not_provided(self) -> None: + service = VectorStoreService( + api_key="test-key", + table_name="test_table", + default_embedding_model="gemini-embedding-2", + default_embed_dim=1536, + ) + + self.assertEqual(service._hnsw_kwargs, DEFAULT_HNSW_KWARGS) + + +if __name__ == "__main__": + unittest.main()