Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions .github/workflows/prod_deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
name: Issueissyu Prod CI/CD

on:
pull_request:
types:
- closed
workflow_dispatch:

permissions:
contents: read

jobs:
deploy:
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' ||
(github.event.pull_request.merged == true && (github.event.pull_request.base.ref == 'prod' || github.event.pull_request.base.ref == 'test/CICD'))

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Create .env file from secret
env:
DEV_ENV: ${{ secrets.PROD_ENV }}
run: |
printf '%s' "$DEV_ENV" > .env
chmod 600 .env

- name: Get current time
uses: josStorer/get-current-time@v2
id: current-time
with:
format: YYYY-MM-DDTHH:mm:ss
utcOffset: "+09:00"

- name: Generate deployment package
run: |
mkdir -p deploy
rsync -a \
--exclude='.git' \
--exclude='deploy' \
--exclude='__pycache__' \
--exclude='.venv' \
--exclude='venv' \
--exclude='.idea' \
--exclude='.cursor' \
--exclude='.github' \
--exclude='*.pyc' \
./ deploy/
(cd deploy && zip -r deploy.zip .)

- name: Deploy to Elastic Beanstalk
uses: einaregilsson/beanstalk-deploy@v22
with:
aws_access_key: ${{ secrets.AWS_PROD_ACCESS_KEY }}
aws_secret_key: ${{ secrets.AWS_PROD_SECRET_KEY }}
application_name: 'issueissyu-ai-prod'
environment_name: 'issueissyu-ai-env'
region: 'ap-northeast-2'
version_label: github-action-${{ steps.current-time.outputs.formattedTime }}
version_description: ${{ github.ref_name }}@${{ github.sha }}
deployment_package: deploy/deploy.zip
wait_for_deployment: false
27 changes: 27 additions & 0 deletions app/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ class Settings(BaseSettings):
aws_secret_key: SecretStr | None = Field(default=None, alias="AWS_SECRET_KEY")
aws_region: str = Field(default="us-east-1", alias="AWS_REGION")
aws_bucket_name: str | None = Field(default=None, alias="AWS_BUCKET")
cdn_enabled: bool = Field(default=False, alias="CDN_ENABLED")
cdn_base_url: str | None = Field(default=None, alias="CDN_BASE_URL")

# SMTP (민원 이메일 실제 송신)
smtp_host: str | None = Field(default=None, alias="SMTP_HOST")
Expand Down Expand Up @@ -158,6 +160,17 @@ class Settings(BaseSettings):
default="simple",
alias="VECTOR_TEXT_SEARCH_CONFIG",
)
vector_hnsw_m: int = Field(default=16, ge=1, alias="VECTOR_HNSW_M")
vector_hnsw_ef_construction: int = Field(
default=64,
ge=1,
alias="VECTOR_HNSW_EF_CONSTRUCTION",
)
vector_hnsw_ef_search: int = Field(default=40, ge=1, alias="VECTOR_HNSW_EF_SEARCH")
vector_hnsw_dist_method: str = Field(
default="vector_cosine_ops",
alias="VECTOR_HNSW_DIST_METHOD",
)

# 문화체육관광부 정책브리핑 정책뉴스 OpenAPI (공공데이터포털)
policy_news_service_key: SecretStr | None = Field(
Expand Down Expand Up @@ -398,6 +411,20 @@ def _empty_string_gemini_embed_batch(cls, value: object) -> object:
return None
return value

@field_validator("cdn_base_url", mode="before")
@classmethod
def _empty_cdn_base_url_to_none(cls, value: object) -> object | None:
if isinstance(value, str) and value.strip() == "":
return None
return value
Comment on lines +414 to +419

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

CDN_BASE_URL이 설정되었을 때, 올바른 URL 형식(예: http:// 또는 https://로 시작)인지 검증하는 로직을 추가하는 것이 안전합니다. 잘못된 형식으로 설정되면 서비스 전체에서 생성되는 퍼블릭 URL이 깨질 수 있습니다.

    @field_validator("cdn_base_url", mode="before")
    @classmethod
    def _empty_cdn_base_url_to_none(cls, value: object) -> object | None:
        if isinstance(value, str):
            val = value.strip()
            if val == "":
                return None
            if not val.startswith(("http://", "https://")):
                raise ValueError("CDN_BASE_URL must start with http:// or https://")
            return val
        return value


@field_validator("vector_hnsw_dist_method", mode="before")
@classmethod
def _empty_vector_hnsw_dist_method(cls, value: object) -> object:
if isinstance(value, str) and value.strip() == "":
return "vector_cosine_ops"
return value

@field_validator(
"local_db_port",
"aws_db_port",
Expand Down
3 changes: 2 additions & 1 deletion app/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from app.services.internal.PolicyPinSchedulerService import PolicyPinSchedulerService
from app.services.internal.complaint_wiring import build_complaint_email_service
from app.services.VectorStoreService import VectorStoreService
from app.services.vector_domains import build_vector_domain_configs
from app.services.vector_domains import build_hnsw_kwargs, build_vector_domain_configs
from app.schemas.IssueDTO import (
CreateIssuePinMultipartRequest,
PinImageIsMainItem,
Expand Down Expand Up @@ -104,6 +104,7 @@ async def lifespan(app: FastAPI):
hybrid_search=settings.vector_hybrid_search,
text_search_config=settings.vector_text_search_config,
embedding_batch_size_override=settings.gemini_embedding_batch_size,
hnsw_kwargs=build_hnsw_kwargs(settings),
)
if settings.vector_dim_check:
dimension_checks = (
Expand Down
65 changes: 63 additions & 2 deletions app/routes/TestRoute.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
from fastapi import APIRouter, File, UploadFile
import io
from pathlib import Path

from fastapi import APIRouter, File, Query, UploadFile
from fastapi.responses import StreamingResponse

from app.core.deps import CurrentUserIdDep, S3UtilDep, UserServiceDep
from app.core.codes import ErrorCode, SuccessCode
from app.core.exceptions import raise_business_exception
from app.core.exceptions import raise_business_exception, raise_file_exception
from app.core.responses import success_response

router = APIRouter(prefix="/test", tags=["test"])

_DEFAULT_S3_TEST_PREFIX = "test-files"


@router.post("/s3/image-upload")
async def upload_test_image_to_s3(
Expand All @@ -17,6 +23,61 @@ async def upload_test_image_to_s3(
return success_response(result=uploaded, success_code=SuccessCode.CREATED)


@router.post("/s3/upload")
async def upload_test_file_to_s3(
s3_util: S3UtilDep,
file: UploadFile = File(...),
prefix: str = Query(default=_DEFAULT_S3_TEST_PREFIX, description="S3 object key prefix"),
):
"""이미지·PDF 등 임의 파일 업로드 테스트. 응답의 key로 다운로드 API를 호출할 수 있습니다."""
raw = await file.read()
if not raw:
raise_file_exception(
ErrorCode.FILE_UPLOAD_ERROR,
detail="업로드할 파일이 비어 있습니다.",
)

filename = (file.filename or "upload.bin").strip() or "upload.bin"
content_type = (file.content_type or "application/octet-stream").split(";")[0].strip()
normalized_prefix = prefix.strip("/") or _DEFAULT_S3_TEST_PREFIX

uploaded = await s3_util.upload_binary(
raw,
filename=filename,
content_type=content_type,
prefix=normalized_prefix,
)
return success_response(result=uploaded, success_code=SuccessCode.CREATED)


@router.get("/s3/download")
async def download_test_file_from_s3(
s3_util: S3UtilDep,
key: str = Query(..., min_length=1, description="S3 object key (업로드 응답의 key)"),
):
"""S3 object key로 파일 다운로드 테스트."""
data, content_type = await s3_util.download_binary(key)
filename = Path(key).name or "download.bin"
return StreamingResponse(
io.BytesIO(data),
media_type=content_type or "application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
Comment on lines +60 to +65

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

다운로드할 파일명에 한글 등 비ASCII 문자가 포함되어 있을 경우, Content-Disposition 헤더의 filename="..." 포맷은 브라우저에 따라 파일명이 깨지거나 헤더 파싱 에러를 유발할 수 있습니다. RFC 5987 표준에 따라 filename*=utf-8''... 포맷을 사용하고 파일명을 퍼센트 인코딩(urllib.parse.quote)하는 것이 안전합니다.

Suggested change
filename = Path(key).name or "download.bin"
return StreamingResponse(
io.BytesIO(data),
media_type=content_type or "application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
from urllib.parse import quote
filename = Path(key).name or "download.bin"
encoded_filename = quote(filename)
return StreamingResponse(
io.BytesIO(data),
media_type=content_type or "application/octet-stream",
headers={"Content-Disposition": f"attachment; filename*=utf-8''{encoded_filename}"},
)



@router.delete("/s3/object")
async def delete_test_file_from_s3(
s3_util: S3UtilDep,
key: str = Query(..., min_length=1, description="삭제할 S3 object key"),
):
"""업로드 테스트 후 S3 객체 삭제."""
deleted = await s3_util.delete_object(key)
return success_response(
result={"key": key, "deleted": deleted},
success_code=SuccessCode.OK,
)


@router.get("/user")
async def get_my_user(user_service: UserServiceDep, uid: CurrentUserIdDep):
user = await user_service.get_user(uid)
Expand Down
8 changes: 4 additions & 4 deletions app/routes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@
ROUTER_REGISTRY = (
{"router": user_router, "disabled_envs": set()},
{"router": issue_router, "disabled_envs": set()},
{"router": complaint_email_router, "disabled_envs": set()},
{"router": complaint_email_router, "disabled_envs": {"prod"}},
{"router": complaint_apply_router, "disabled_envs": set()},
{"router": image_geo_router, "disabled_envs": set()},
{"router": image_geo_router, "disabled_envs": {"prod"}},
{"router": test_router, "disabled_envs": {"dev", "prod"}},
{"router": vector_test_router, "disabled_envs": {"dev", "prod"}},
{"router": festival_pin_router, "disabled_envs": {"prod"}},
{"router": festival_admin_router, "disabled_envs": set()},
{"router": contest_pin_router, "disabled_envs": {"prod"}},
{"router": contest_admin_router, "disabled_envs": {"prod"}},
{"router": contest_admin_router, "disabled_envs": set()},
{"router": policy_pin_router, "disabled_envs": {"prod"}},
{"router": policy_admin_router, "disabled_envs": {"prod"}},
{"router": policy_admin_router, "disabled_envs": set()},
)


Expand Down
17 changes: 11 additions & 6 deletions app/services/VectorStoreService.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@

logger = logging.getLogger(__name__)

DEFAULT_HNSW_KWARGS: dict[str, Any] = {
"hnsw_m": 16,
"hnsw_ef_construction": 64,
"hnsw_ef_search": 40,
"hnsw_dist_method": "vector_cosine_ops",
}

@dataclass(slots=True)
class _VectorIndexBundle:
table_name: str
Expand All @@ -39,6 +46,7 @@ def __init__(
hybrid_search: bool = True,
text_search_config: str = "simple",
embedding_batch_size_override: int | None = None,
hnsw_kwargs: dict[str, Any] | None = None,
) -> None:
self._sync_database_url = sync_database_url
self._async_database_url = async_database_url
Expand All @@ -52,6 +60,8 @@ def __init__(
self._domain_configs = domain_configs or {}
self._embed_models: dict[tuple[str, int], BaseEmbedding] = {}
self._embedding_batch_size_override = embedding_batch_size_override
self._hnsw_kwargs = hnsw_kwargs or dict(DEFAULT_HNSW_KWARGS)
logger.info("Vector HNSW config: %s", self._hnsw_kwargs)
Comment on lines +63 to +64

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

hnsw_kwargs가 제공될 때 DEFAULT_HNSW_KWARGS를 완전히 대체(override)하게 됩니다. 만약 호출부에서 일부 키(예: {"hnsw_m": 24})만 전달할 경우, 나머지 필수 설정 키들이 누락되어 에러가 발생하거나 기본값이 적용되지 않을 수 있습니다. 기본값 딕셔너리와 입력받은 딕셔너리를 병합(merge)하도록 수정하는 것이 안전합니다.

Suggested change
self._hnsw_kwargs = hnsw_kwargs or dict(DEFAULT_HNSW_KWARGS)
logger.info("Vector HNSW config: %s", self._hnsw_kwargs)
self._hnsw_kwargs = {**DEFAULT_HNSW_KWARGS, **(hnsw_kwargs or {})}
logger.info("Vector HNSW config: %s", self._hnsw_kwargs)


@staticmethod
def _normalize_domain(domain: str) -> str:
Expand Down Expand Up @@ -123,12 +133,7 @@ def _get_or_create_bundle(
embed_dim=embed_dim,
hybrid_search=self._hybrid_search,
text_search_config=self._text_search_config,
hnsw_kwargs={
"hnsw_m": 16,
"hnsw_ef_construction": 64,
"hnsw_ef_search": 40,
"hnsw_dist_method": "vector_cosine_ops",
},
hnsw_kwargs=self._hnsw_kwargs,
)
bundle = _VectorIndexBundle(
table_name=resolved_table_name,
Expand Down
10 changes: 10 additions & 0 deletions app/services/vector_domains.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from dataclasses import dataclass
from enum import Enum
from typing import Any

from app.core.config import Settings

Expand Down Expand Up @@ -51,3 +52,12 @@ def build_vector_domain_configs(settings: Settings) -> dict[VectorDomain, Domain
standalone_table=False,
),
}


def build_hnsw_kwargs(settings: Settings) -> dict[str, Any]:
return {
"hnsw_m": settings.vector_hnsw_m,
"hnsw_ef_construction": settings.vector_hnsw_ef_construction,
"hnsw_ef_search": settings.vector_hnsw_ef_search,
"hnsw_dist_method": settings.vector_hnsw_dist_method,
}
3 changes: 3 additions & 0 deletions app/utils/S3Util.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ def _build_public_file_url(self, object_key: str) -> str:
bucket_name = self._ensure_bucket_name()
resolved_region = self.region_name or "us-east-1"
encoded_key = quote(object_key.lstrip("/"), safe="/")
cdn_base_url = (settings.cdn_base_url or "").strip()
if settings.cdn_enabled and cdn_base_url:
return f"{cdn_base_url.rstrip('/')}/{encoded_key}"
Comment on lines +108 to +110

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

CDN_ENABLED가 True로 설정되어 있지만 CDN_BASE_URL이 비어 있는 경우, 의도치 않은 설정 오류일 가능성이 높습니다. 이 경우 S3 URL로 폴백하기 전에 경고 로그를 남겨두면 운영 환경에서 문제를 디버깅하는 데 큰 도움이 됩니다.

Suggested change
cdn_base_url = (settings.cdn_base_url or "").strip()
if settings.cdn_enabled and cdn_base_url:
return f"{cdn_base_url.rstrip('/')}/{encoded_key}"
cdn_base_url = (settings.cdn_base_url or "").strip()
if settings.cdn_enabled:
if cdn_base_url:
return f"{cdn_base_url.rstrip('/')}/{encoded_key}"
logger.warning("CDN_ENABLED is True but CDN_BASE_URL is not configured. Falling back to S3 URL.")

if resolved_region == "us-east-1":
return f"https://{bucket_name}.s3.amazonaws.com/{encoded_key}"
return f"https://{bucket_name}.s3.{resolved_region}.amazonaws.com/{encoded_key}"
Expand Down
3 changes: 2 additions & 1 deletion rag/scripts/insert_chunks_to_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# python -m rag.scripts.insert_chunks_to_vector 형태 실행 기준
from app.core.config import settings
from app.services.VectorStoreService import VectorStoreService
from app.services.vector_domains import DomainVectorConfig, VectorDomain
from app.services.vector_domains import DomainVectorConfig, VectorDomain, build_hnsw_kwargs
from app.utils.chunk_node_metadata import build_chunk_metadata
from app.utils.chunk_text_normalize import load_skip_line_prefixes, normalize_chunk
from rag.scripts.chunk_module import iter_jsonl
Expand Down Expand Up @@ -44,6 +44,7 @@ def build_service() -> VectorStoreService:
hybrid_search=settings.vector_hybrid_search,
text_search_config=settings.vector_text_search_config,
embedding_batch_size_override=settings.gemini_embedding_batch_size,
hnsw_kwargs=build_hnsw_kwargs(settings),
)


Expand Down
41 changes: 41 additions & 0 deletions tests/test_router_registry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
from __future__ import annotations

import unittest

from app.routes import get_enabled_routers


def _router_tags(env: str) -> set[str]:
return {router.tags[0] for router in get_enabled_routers(env) if router.tags}


class RouterRegistryTest(unittest.TestCase):
def test_prod_excludes_pipeline_and_debug_routes(self) -> None:
tags = _router_tags("prod")

self.assertNotIn("complaint-email", tags)
self.assertNotIn("geo", tags)
self.assertNotIn("festival-pins", tags)
self.assertNotIn("contest-pins", tags)
self.assertNotIn("policy-pins", tags)
self.assertNotIn("test", tags)
self.assertNotIn("vector-test", tags)

def test_prod_includes_core_and_admin_routes(self) -> None:
tags = _router_tags("prod")

self.assertIn("issue", tags)
self.assertIn("complaint-apply", tags)
self.assertIn("festival-admin", tags)
self.assertIn("contest-admin", tags)
self.assertIn("policy-admin", tags)
self.assertIn("users", tags)

def test_dev_includes_complaint_email_and_geo(self) -> None:
tags = _router_tags("dev")

self.assertIn("complaint-email", tags)
self.assertIn("geo", tags)
self.assertIn("festival-pins", tags)
self.assertIn("contest-admin", tags)
self.assertIn("policy-admin", tags)
Loading
Loading