Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions .github/workflows/prod_deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
name: Issueissyu Prod CI/CD

on:
pull_request:
types:
- closed
workflow_dispatch:

permissions:
contents: read

jobs:
deploy:
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' ||
(github.event.pull_request.merged == true && (github.event.pull_request.base.ref == 'prod' || github.event.pull_request.base.ref == 'test/CICD'))

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Create .env file from secret
env:
DEV_ENV: ${{ secrets.PROD_ENV }}
run: |
printf '%s' "$DEV_ENV" > .env
chmod 600 .env

- name: Get current time
uses: josStorer/get-current-time@v2
id: current-time
with:
format: YYYY-MM-DDTHH:mm:ss
utcOffset: "+09:00"

- name: Generate deployment package
run: |
mkdir -p deploy
rsync -a \
--exclude='.git' \
--exclude='deploy' \
--exclude='__pycache__' \
--exclude='.venv' \
--exclude='venv' \
--exclude='.idea' \
--exclude='.cursor' \
--exclude='.github' \
--exclude='*.pyc' \
./ deploy/
(cd deploy && zip -r deploy.zip .)

- name: Deploy to Elastic Beanstalk
uses: einaregilsson/beanstalk-deploy@v22
with:
aws_access_key: ${{ secrets.AWS_PROD_ACCESS_KEY }}
aws_secret_key: ${{ secrets.AWS_PROD_SECRET_KEY }}
application_name: 'issueissyu-ai-prod'
environment_name: 'issueissyu-ai-env'
region: 'ap-northeast-2'
version_label: github-action-${{ steps.current-time.outputs.formattedTime }}
version_description: ${{ github.ref_name }}@${{ github.sha }}
deployment_package: deploy/deploy.zip
wait_for_deployment: false
27 changes: 27 additions & 0 deletions app/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ class Settings(BaseSettings):
aws_secret_key: SecretStr | None = Field(default=None, alias="AWS_SECRET_KEY")
aws_region: str = Field(default="us-east-1", alias="AWS_REGION")
aws_bucket_name: str | None = Field(default=None, alias="AWS_BUCKET")
cdn_enabled: bool = Field(default=False, alias="CDN_ENABLED")
cdn_base_url: str | None = Field(default=None, alias="CDN_BASE_URL")

# SMTP (민원 이메일 실제 송신)
smtp_host: str | None = Field(default=None, alias="SMTP_HOST")
Expand Down Expand Up @@ -158,6 +160,17 @@ class Settings(BaseSettings):
default="simple",
alias="VECTOR_TEXT_SEARCH_CONFIG",
)
vector_hnsw_m: int = Field(default=16, ge=1, alias="VECTOR_HNSW_M")
vector_hnsw_ef_construction: int = Field(
default=64,
ge=1,
alias="VECTOR_HNSW_EF_CONSTRUCTION",
)
vector_hnsw_ef_search: int = Field(default=40, ge=1, alias="VECTOR_HNSW_EF_SEARCH")
vector_hnsw_dist_method: str = Field(
default="vector_cosine_ops",
alias="VECTOR_HNSW_DIST_METHOD",
)

# 문화체육관광부 정책브리핑 정책뉴스 OpenAPI (공공데이터포털)
policy_news_service_key: SecretStr | None = Field(
Expand Down Expand Up @@ -398,6 +411,20 @@ def _empty_string_gemini_embed_batch(cls, value: object) -> object:
return None
return value

@field_validator("cdn_base_url", mode="before")
@classmethod
def _empty_cdn_base_url_to_none(cls, value: object) -> object | None:
if isinstance(value, str) and value.strip() == "":
return None
return value

@field_validator("vector_hnsw_dist_method", mode="before")
@classmethod
def _empty_vector_hnsw_dist_method(cls, value: object) -> object:
if isinstance(value, str) and value.strip() == "":
return "vector_cosine_ops"
return value
Comment on lines +423 to +426

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

vector_hnsw_dist_method 필드는 str 타입으로 정의되어 있으므로, 만약 환경 변수나 설정 값에서 None이 명시적으로 전달될 경우 isinstance(value, str) 조건에 걸리지 않아 None을 그대로 반환하게 됩니다. 이 경우 Pydantic 검증 단계에서 ValidationError가 발생할 수 있습니다.

안전한 방어적 프로그래밍을 위해 valueNone인 경우에도 기본값인 "vector_cosine_ops"를 반환하도록 처리하는 것이 좋습니다.

Suggested change
def _empty_vector_hnsw_dist_method(cls, value: object) -> object:
if isinstance(value, str) and value.strip() == "":
return "vector_cosine_ops"
return value
def _empty_vector_hnsw_dist_method(cls, value: object) -> object:
if value is None or (isinstance(value, str) and value.strip() == ""):
return "vector_cosine_ops"
return value


@field_validator(
"local_db_port",
"aws_db_port",
Expand Down
3 changes: 2 additions & 1 deletion app/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from app.services.internal.PolicyPinSchedulerService import PolicyPinSchedulerService
from app.services.internal.complaint_wiring import build_complaint_email_service
from app.services.VectorStoreService import VectorStoreService
from app.services.vector_domains import build_vector_domain_configs
from app.services.vector_domains import build_hnsw_kwargs, build_vector_domain_configs
from app.schemas.IssueDTO import (
CreateIssuePinMultipartRequest,
PinImageIsMainItem,
Expand Down Expand Up @@ -104,6 +104,7 @@ async def lifespan(app: FastAPI):
hybrid_search=settings.vector_hybrid_search,
text_search_config=settings.vector_text_search_config,
embedding_batch_size_override=settings.gemini_embedding_batch_size,
hnsw_kwargs=build_hnsw_kwargs(settings),
)
if settings.vector_dim_check:
dimension_checks = (
Expand Down
65 changes: 63 additions & 2 deletions app/routes/TestRoute.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
from fastapi import APIRouter, File, UploadFile
import io
from pathlib import Path

from fastapi import APIRouter, File, Query, UploadFile
from fastapi.responses import StreamingResponse

from app.core.deps import CurrentUserIdDep, S3UtilDep, UserServiceDep
from app.core.codes import ErrorCode, SuccessCode
from app.core.exceptions import raise_business_exception
from app.core.exceptions import raise_business_exception, raise_file_exception
from app.core.responses import success_response

router = APIRouter(prefix="/test", tags=["test"])

_DEFAULT_S3_TEST_PREFIX = "test-files"


@router.post("/s3/image-upload")
async def upload_test_image_to_s3(
Expand All @@ -17,6 +23,61 @@ async def upload_test_image_to_s3(
return success_response(result=uploaded, success_code=SuccessCode.CREATED)


@router.post("/s3/upload")
async def upload_test_file_to_s3(
s3_util: S3UtilDep,
file: UploadFile = File(...),
prefix: str = Query(default=_DEFAULT_S3_TEST_PREFIX, description="S3 object key prefix"),
):
"""이미지·PDF 등 임의 파일 업로드 테스트. 응답의 key로 다운로드 API를 호출할 수 있습니다."""
raw = await file.read()
if not raw:
raise_file_exception(
ErrorCode.FILE_UPLOAD_ERROR,
detail="업로드할 파일이 비어 있습니다.",
)

filename = (file.filename or "upload.bin").strip() or "upload.bin"
content_type = (file.content_type or "application/octet-stream").split(";")[0].strip()
normalized_prefix = prefix.strip("/") or _DEFAULT_S3_TEST_PREFIX

uploaded = await s3_util.upload_binary(
raw,
filename=filename,
content_type=content_type,
prefix=normalized_prefix,
)
return success_response(result=uploaded, success_code=SuccessCode.CREATED)


@router.get("/s3/download")
async def download_test_file_from_s3(
s3_util: S3UtilDep,
key: str = Query(..., min_length=1, description="S3 object key (업로드 응답의 key)"),
):
"""S3 object key로 파일 다운로드 테스트."""
data, content_type = await s3_util.download_binary(key)
filename = Path(key).name or "download.bin"
return StreamingResponse(
io.BytesIO(data),
media_type=content_type or "application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
Comment on lines +60 to +65

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

S3에서 다운로드할 파일의 이름(filename)에 한글 등 비ASCII 문자가 포함되어 있을 경우, Content-Disposition 헤더에 그대로 설정하면 ASGI 서버(Uvicorn 등)에서 UnicodeEncodeError가 발생하여 다운로드 요청이 실패(500 Internal Server Error)할 수 있습니다.

RFC 5987 및 RFC 6266 표준에 따라 파일명을 URL 인코딩(urllib.parse.quote)하고, filename*=UTF-8'' 형식을 사용하여 안전하게 전달하는 것을 권장합니다.

    filename = Path(key).name or "download.bin"
    import urllib.parse
    encoded_filename = urllib.parse.quote(filename)
    return StreamingResponse(
        io.BytesIO(data),
        media_type=content_type or "application/octet-stream",
        headers={
            "Content-Disposition": f"attachment; filename=\"{encoded_filename}\"; filename*=UTF-8''{encoded_filename}"
        },
    )



@router.delete("/s3/object")
async def delete_test_file_from_s3(
s3_util: S3UtilDep,
key: str = Query(..., min_length=1, description="삭제할 S3 object key"),
):
"""업로드 테스트 후 S3 객체 삭제."""
deleted = await s3_util.delete_object(key)
return success_response(
result={"key": key, "deleted": deleted},
success_code=SuccessCode.OK,
)


@router.get("/user")
async def get_my_user(user_service: UserServiceDep, uid: CurrentUserIdDep):
user = await user_service.get_user(uid)
Expand Down
8 changes: 4 additions & 4 deletions app/routes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@
ROUTER_REGISTRY = (
{"router": user_router, "disabled_envs": set()},
{"router": issue_router, "disabled_envs": set()},
{"router": complaint_email_router, "disabled_envs": set()},
{"router": complaint_email_router, "disabled_envs": {"prod"}},
{"router": complaint_apply_router, "disabled_envs": set()},
{"router": image_geo_router, "disabled_envs": set()},
{"router": image_geo_router, "disabled_envs": {"prod"}},
{"router": test_router, "disabled_envs": {"dev", "prod"}},
{"router": vector_test_router, "disabled_envs": {"dev", "prod"}},
{"router": festival_pin_router, "disabled_envs": {"prod"}},
{"router": festival_admin_router, "disabled_envs": set()},
{"router": contest_pin_router, "disabled_envs": {"prod"}},
{"router": contest_admin_router, "disabled_envs": {"prod"}},
{"router": contest_admin_router, "disabled_envs": set()},
{"router": policy_pin_router, "disabled_envs": {"prod"}},
{"router": policy_admin_router, "disabled_envs": {"prod"}},
{"router": policy_admin_router, "disabled_envs": set()},
)


Expand Down
17 changes: 11 additions & 6 deletions app/services/VectorStoreService.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@

logger = logging.getLogger(__name__)

DEFAULT_HNSW_KWARGS: dict[str, Any] = {
"hnsw_m": 16,
"hnsw_ef_construction": 64,
"hnsw_ef_search": 40,
"hnsw_dist_method": "vector_cosine_ops",
}

@dataclass(slots=True)
class _VectorIndexBundle:
table_name: str
Expand All @@ -39,6 +46,7 @@ def __init__(
hybrid_search: bool = True,
text_search_config: str = "simple",
embedding_batch_size_override: int | None = None,
hnsw_kwargs: dict[str, Any] | None = None,
) -> None:
self._sync_database_url = sync_database_url
self._async_database_url = async_database_url
Expand All @@ -52,6 +60,8 @@ def __init__(
self._domain_configs = domain_configs or {}
self._embed_models: dict[tuple[str, int], BaseEmbedding] = {}
self._embedding_batch_size_override = embedding_batch_size_override
self._hnsw_kwargs = hnsw_kwargs or dict(DEFAULT_HNSW_KWARGS)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

hnsw_kwargs가 전달될 때, 일부 키만 포함된 부분 딕셔너리(예: {"hnsw_m": 32})가 전달되면 나머지 필수 키(hnsw_ef_construction, hnsw_ef_search, hnsw_dist_method 등)가 누락되어 에러가 발생하거나 기본값이 적용되지 않을 수 있습니다.

DEFAULT_HNSW_KWARGS를 기반으로 입력받은 hnsw_kwargs를 병합(merge)하여 항상 모든 필수 키가 존재하도록 보장하는 것이 안전합니다.

Suggested change
self._hnsw_kwargs = hnsw_kwargs or dict(DEFAULT_HNSW_KWARGS)
self._hnsw_kwargs = {**DEFAULT_HNSW_KWARGS, **(hnsw_kwargs or {})}

logger.info("Vector HNSW config: %s", self._hnsw_kwargs)

@staticmethod
def _normalize_domain(domain: str) -> str:
Expand Down Expand Up @@ -123,12 +133,7 @@ def _get_or_create_bundle(
embed_dim=embed_dim,
hybrid_search=self._hybrid_search,
text_search_config=self._text_search_config,
hnsw_kwargs={
"hnsw_m": 16,
"hnsw_ef_construction": 64,
"hnsw_ef_search": 40,
"hnsw_dist_method": "vector_cosine_ops",
},
hnsw_kwargs=self._hnsw_kwargs,
)
bundle = _VectorIndexBundle(
table_name=resolved_table_name,
Expand Down
10 changes: 10 additions & 0 deletions app/services/vector_domains.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from dataclasses import dataclass
from enum import Enum
from typing import Any

from app.core.config import Settings

Expand Down Expand Up @@ -51,3 +52,12 @@ def build_vector_domain_configs(settings: Settings) -> dict[VectorDomain, Domain
standalone_table=False,
),
}


def build_hnsw_kwargs(settings: Settings) -> dict[str, Any]:
return {
"hnsw_m": settings.vector_hnsw_m,
"hnsw_ef_construction": settings.vector_hnsw_ef_construction,
"hnsw_ef_search": settings.vector_hnsw_ef_search,
"hnsw_dist_method": settings.vector_hnsw_dist_method,
}
3 changes: 3 additions & 0 deletions app/utils/S3Util.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ def _build_public_file_url(self, object_key: str) -> str:
bucket_name = self._ensure_bucket_name()
resolved_region = self.region_name or "us-east-1"
encoded_key = quote(object_key.lstrip("/"), safe="/")
cdn_base_url = (settings.cdn_base_url or "").strip()
if settings.cdn_enabled and cdn_base_url:
return f"{cdn_base_url.rstrip('/')}/{encoded_key}"
if resolved_region == "us-east-1":
return f"https://{bucket_name}.s3.amazonaws.com/{encoded_key}"
return f"https://{bucket_name}.s3.{resolved_region}.amazonaws.com/{encoded_key}"
Expand Down
3 changes: 2 additions & 1 deletion rag/scripts/insert_chunks_to_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# python -m rag.scripts.insert_chunks_to_vector 형태 실행 기준
from app.core.config import settings
from app.services.VectorStoreService import VectorStoreService
from app.services.vector_domains import DomainVectorConfig, VectorDomain
from app.services.vector_domains import DomainVectorConfig, VectorDomain, build_hnsw_kwargs
from app.utils.chunk_node_metadata import build_chunk_metadata
from app.utils.chunk_text_normalize import load_skip_line_prefixes, normalize_chunk
from rag.scripts.chunk_module import iter_jsonl
Expand Down Expand Up @@ -44,6 +44,7 @@ def build_service() -> VectorStoreService:
hybrid_search=settings.vector_hybrid_search,
text_search_config=settings.vector_text_search_config,
embedding_batch_size_override=settings.gemini_embedding_batch_size,
hnsw_kwargs=build_hnsw_kwargs(settings),
)


Expand Down
41 changes: 41 additions & 0 deletions tests/test_router_registry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
from __future__ import annotations

import unittest

from app.routes import get_enabled_routers


def _router_tags(env: str) -> set[str]:
return {router.tags[0] for router in get_enabled_routers(env) if router.tags}


class RouterRegistryTest(unittest.TestCase):
def test_prod_excludes_pipeline_and_debug_routes(self) -> None:
tags = _router_tags("prod")

self.assertNotIn("complaint-email", tags)
self.assertNotIn("geo", tags)
self.assertNotIn("festival-pins", tags)
self.assertNotIn("contest-pins", tags)
self.assertNotIn("policy-pins", tags)
self.assertNotIn("test", tags)
self.assertNotIn("vector-test", tags)

def test_prod_includes_core_and_admin_routes(self) -> None:
tags = _router_tags("prod")

self.assertIn("issue", tags)
self.assertIn("complaint-apply", tags)
self.assertIn("festival-admin", tags)
self.assertIn("contest-admin", tags)
self.assertIn("policy-admin", tags)
self.assertIn("users", tags)

def test_dev_includes_complaint_email_and_geo(self) -> None:
tags = _router_tags("dev")

self.assertIn("complaint-email", tags)
self.assertIn("geo", tags)
self.assertIn("festival-pins", tags)
self.assertIn("contest-admin", tags)
self.assertIn("policy-admin", tags)
Loading
Loading