diff --git a/.template.yaml b/.template.yaml index f8c8b1fc..d501d481 100644 --- a/.template.yaml +++ b/.template.yaml @@ -1,5 +1,5 @@ # Autogenerated - DO NOT EDIT # Parameters of the project as generated from template -_commit: 33b991b +_commit: 179dc2a _src_path: gh:kraysent/python-template project_name: db_app diff --git a/app/data/repositories/common.py b/app/data/repositories/common.py index 5306e573..3a83f8b9 100644 --- a/app/data/repositories/common.py +++ b/app/data/repositories/common.py @@ -6,7 +6,6 @@ from app.data import model, template from app.lib import concurrency from app.lib.storage import postgres -from app.lib.web.errors import DatabaseError @dataclass @@ -40,9 +39,6 @@ def create_bibliography(self, code: str, year: int, authors: list[str], title: s params=[code, year, authors, title], ) - if result is None: - raise DatabaseError("no result returned from query") - return int(result["id"]) def get_source_entry(self, source_name: str) -> model.Bibliography: @@ -77,7 +73,7 @@ def get_schema(self, schema_name: str, table_name: str) -> TableSchemaInfo: table_row = table_task.result() table_description = "" - if table_row is not None and table_row.get("param") is not None: + if table_row.get("param") is not None: param = table_row["param"] if isinstance(param, dict): table_description = param.get("description") or "" diff --git a/app/data/repositories/layer0/records.py b/app/data/repositories/layer0/records.py index fa4f657e..5e37fd44 100644 --- a/app/data/repositories/layer0/records.py +++ b/app/data/repositories/layer0/records.py @@ -35,8 +35,6 @@ def register_records(self, table_name: str, record_ids: list[str]) -> None: raise RuntimeError("no records to upsert") table_id_row = self._storage.query_one(template.FETCH_RAWDATA_REGISTRY, params=[table_name]) - if table_id_row is None: - raise DatabaseError(f"unable to fetch table with name {table_name}") table_id = table_id_row["id"] @@ -121,8 +119,6 @@ def get_processed_records( def get_table_statistics(self, table_name: str) -> model.TableStatistics: table_id_row = self._storage.query_one(template.FETCH_RAWDATA_REGISTRY, params=[table_name]) - if table_id_row is None: - raise DatabaseError(f"unable to fetch table with name {table_name}") table_id = table_id_row["id"] diff --git a/app/data/repositories/layer0/tables.py b/app/data/repositories/layer0/tables.py index ce2eed10..4b15738f 100644 --- a/app/data/repositories/layer0/tables.py +++ b/app/data/repositories/layer0/tables.py @@ -14,7 +14,6 @@ from app.data import model, repositories, template from app.data.repositories.layer0.common import INTERNAL_ID_COLUMN_NAME, RAWDATA_SCHEMA from app.lib.storage import postgres -from app.lib.web.errors import DatabaseError log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -419,8 +418,6 @@ def fetch_metadata(self, table_name: str) -> model.Layer0TableMeta: def fetch_metadata_by_name(self, table_name: str) -> model.Layer0TableMeta: row = self._storage.query_one(template.FETCH_RAWDATA_REGISTRY, params=[table_name]) - if row is None: - raise DatabaseError(f"unable to fetch table with name {table_name}") modification_dt: datetime.datetime | None = row.get("modification_dt") return self._fetch_metadata_by_name(table_name, modification_dt) @@ -455,9 +452,6 @@ def _fetch_metadata_by_name( table_metadata = self._storage.query_one(template.FETCH_TABLE_METADATA, params=[RAWDATA_SCHEMA, table_name]) registry_item = self._storage.query_one(template.FETCH_RAWDATA_REGISTRY, params=[table_name]) - if table_metadata is None: - raise DatabaseError(f"unable to metadata for table {table_name}") - return model.Layer0TableMeta( table_name, descriptions, diff --git a/app/domain/responders/fits_responder.py b/app/domain/responders/fits_responder.py index abc3d015..9b34dcd6 100644 --- a/app/domain/responders/fits_responder.py +++ b/app/domain/responders/fits_responder.py @@ -18,14 +18,10 @@ def extract_object_data(objects: list[model.Layer2CatalogObject]) -> dict[str, n catalog_name = catalog_obj.catalog().value catalog_data = catalog_obj.layer2_data() - if isinstance(catalog_data, dict): - for field, _ in catalog_data.items(): - full_field_name = f"{catalog_name}_{field}" - if full_field_name not in data_dict: - data_dict[full_field_name] = [] - else: - if catalog_name not in data_dict: - data_dict[catalog_name] = [] + for field, _ in catalog_data.items(): + full_field_name = f"{catalog_name}_{field}" + if full_field_name not in data_dict: + data_dict[full_field_name] = [] for obj in objects: for field in data_dict: @@ -37,12 +33,9 @@ def extract_object_data(objects: list[model.Layer2CatalogObject]) -> dict[str, n catalog_name = catalog_obj.catalog().value catalog_data = catalog_obj.layer2_data() - if isinstance(catalog_data, dict): - for field, value in catalog_data.items(): - full_field_name = f"{catalog_name}_{field}" - data_dict[full_field_name][-1] = value - else: - data_dict[catalog_name][-1] = catalog_data + for field, value in catalog_data.items(): + full_field_name = f"{catalog_name}_{field}" + data_dict[full_field_name][-1] = value for field, values in data_dict.items(): if all(v is None for v in values): diff --git a/app/domain/responders/json_responder.py b/app/domain/responders/json_responder.py index 033e969a..e5b2d2b0 100644 --- a/app/domain/responders/json_responder.py +++ b/app/domain/responders/json_responder.py @@ -15,5 +15,5 @@ def objects_to_response(objects: list[model.Layer2CatalogObject]) -> list[dataap class JSONResponder(interface.ObjectResponder): - def build_response(self, objects: list[model.Layer2CatalogObject]) -> Any: + def build_response_from_catalog(self, objects: list[model.Layer2CatalogObject]) -> Any: return objects_to_response(objects) diff --git a/app/lib/astronomy/__init__.py b/app/lib/astronomy/__init__.py index 2eb83ac6..b2632489 100644 --- a/app/lib/astronomy/__init__.py +++ b/app/lib/astronomy/__init__.py @@ -3,7 +3,7 @@ from astropy import constants from astropy import units as u from uncertainties import ufloat -from uncertainties.umath import cos, sin # type: ignore +from uncertainties.umath import cos, sin warnings.filterwarnings("ignore", message="Using UFloat objects with std_dev==0 may give unexpected results") @@ -72,7 +72,7 @@ def velocity_wr_apex( lat_apex_u = ufloat(lat_apex.value, lat_apex_err_val) result = vel_u - vel_apex_u * ( - sin(lat_u) * sin(lat_apex_u) + cos(lat_u) * cos(lat_apex_u) * cos(lon_u - lon_apex_u) # type: ignore + sin(lat_u) * sin(lat_apex_u) + cos(lat_u) * cos(lat_apex_u) * cos(lon_u - lon_apex_u) ) return u.Quantity(result.nominal_value, unit="km/s"), u.Quantity(result.std_dev, unit="km/s") diff --git a/app/presentation/adminapi/interface.py b/app/presentation/adminapi/interface.py index 79b60afa..4bd8c134 100644 --- a/app/presentation/adminapi/interface.py +++ b/app/presentation/adminapi/interface.py @@ -338,49 +338,49 @@ class SetCrossmatchResultsResponse(pydantic.BaseModel): class Actions(abc.ABC): @abc.abstractmethod - def add_data(self, request: AddDataRequest) -> AddDataResponse: + def add_data(self, r: AddDataRequest) -> AddDataResponse: pass @abc.abstractmethod - def create_table(self, request: CreateTableRequest) -> tuple[CreateTableResponse, bool]: + def create_table(self, r: CreateTableRequest) -> tuple[CreateTableResponse, bool]: pass @abc.abstractmethod - def get_table(self, request: GetTableRequest) -> GetTableResponse: + def get_table(self, r: GetTableRequest) -> GetTableResponse: pass @abc.abstractmethod - def get_table_list(self, request: GetTableListRequest) -> GetTableListResponse: + def get_table_list(self, r: GetTableListRequest) -> GetTableListResponse: pass @abc.abstractmethod - def patch_table(self, request: PatchTableRequest) -> PatchTableResponse: + def patch_table(self, r: PatchTableRequest) -> PatchTableResponse: pass @abc.abstractmethod - def create_source(self, request: CreateSourceRequest) -> CreateSourceResponse: + def create_source(self, r: CreateSourceRequest) -> CreateSourceResponse: pass @abc.abstractmethod - def login(self, request: LoginRequest) -> LoginResponse: + def login(self, r: LoginRequest) -> LoginResponse: pass @abc.abstractmethod - def get_records(self, request: GetRecordsRequest) -> GetRecordsResponse: + def get_records(self, r: GetRecordsRequest) -> GetRecordsResponse: pass @abc.abstractmethod - def get_crossmatch_records(self, request: GetRecordsCrossmatchRequest) -> GetRecordsCrossmatchResponse: + def get_crossmatch_records(self, r: GetRecordsCrossmatchRequest) -> GetRecordsCrossmatchResponse: pass @abc.abstractmethod - def get_record_crossmatch(self, request: GetRecordCrossmatchRequest) -> GetRecordCrossmatchResponse: + def get_record_crossmatch(self, r: GetRecordCrossmatchRequest) -> GetRecordCrossmatchResponse: pass @abc.abstractmethod - def save_structured_data(self, request: SaveStructuredDataRequest) -> SaveStructuredDataResponse: + def save_structured_data(self, r: SaveStructuredDataRequest) -> SaveStructuredDataResponse: pass @abc.abstractmethod - def set_crossmatch_results(self, request: SetCrossmatchResultsRequest) -> SetCrossmatchResultsResponse: + def set_crossmatch_results(self, r: SetCrossmatchResultsRequest) -> SetCrossmatchResultsResponse: pass diff --git a/app/presentation/adminapi/server.py b/app/presentation/adminapi/server.py index 5eb72653..adcbe5cf 100644 --- a/app/presentation/adminapi/server.py +++ b/app/presentation/adminapi/server.py @@ -18,7 +18,6 @@ def __init__(self, actions: interface.Actions) -> None: def add_data( self, request: interface.AddDataRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.AddDataResponse]: response = self.actions.add_data(request) return server.APIOkResponse(data=response) @@ -26,7 +25,6 @@ def add_data( def create_source( self, request: interface.CreateSourceRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.CreateSourceResponse]: response = self.actions.create_source(request) return server.APIOkResponse(data=response) @@ -34,7 +32,6 @@ def create_source( def create_table( self, request: interface.CreateTableRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.CreateTableResponse]: response, _ = self.actions.create_table(request) return server.APIOkResponse(data=response) @@ -60,7 +57,6 @@ def get_records( def patch_table( self, request: interface.PatchTableRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.PatchTableResponse]: response = self.actions.patch_table(request) return server.APIOkResponse(data=response) @@ -84,7 +80,6 @@ def get_record_crossmatch( def save_structured_data( self, request: interface.SaveStructuredDataRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.SaveStructuredDataResponse]: response = self.actions.save_structured_data(request) return server.APIOkResponse(data=response) @@ -92,7 +87,6 @@ def save_structured_data( def set_crossmatch_results( self, request: interface.SetCrossmatchResultsRequest, - token: str = fastapi.Security(api_key_header), ) -> server.APIOkResponse[interface.SetCrossmatchResultsResponse]: response = self.actions.set_crossmatch_results(request) return server.APIOkResponse(data=response) diff --git a/makefile b/makefile index e20ac83d..2185a346 100644 --- a/makefile +++ b/makefile @@ -13,27 +13,41 @@ check: else \ echo "$$output"; \ fi + @find . \ -name "*.py" \ -not -path "./.venv/*" \ -not -path "./.git/*" \ -exec uv run python -m py_compile {} + + @echo "Compilation ok." + @uvx ruff format \ --quiet \ --config=pyproject.toml \ --check + @echo "Formatting ok." + @uvx ruff check \ --quiet \ --config=pyproject.toml + @echo "Linter ok." + + @output=$$(uvx basedpyright 2>&1); exit_code=$$?; \ + if [ $$exit_code -ne 0 ]; then echo "$$output"; fi; \ + exit $$exit_code + @echo "Typechecking ok." + @uv run pytest \ --quiet \ --config-file=pyproject.toml \ tests/env_test.py tests/unit + @echo "Testing ok." fix: @uvx ruff format \ --quiet \ --config=pyproject.toml + @uvx ruff check \ --quiet \ --config=pyproject.toml \ @@ -109,7 +123,7 @@ build-docs: cleanup: rm -rf uv.lock .venv \ - .pytest_cache .mypy_cache .vizier_cache .ruff_cache \ + .pytest_cache .ruff_cache \ __pycache__ */__pycache__ \ .coverage htmlcov site \ docs/gen @@ -126,10 +140,6 @@ test-all: check test-regression: uv run tests.py regression-tests -mypy: - uvx mypy app --config-file pyproject.toml - uvx mypy tests --config-file pyproject.toml - coverage: uvx coverage run -m unittest discover -s tests -p "*_test.py" -v uvx coverage html diff --git a/pyproject.toml b/pyproject.toml index dd1d7610..75f649d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,24 +32,22 @@ dependencies = [ ] [dependency-groups] -dev = [ - "pytest>=8.3.4", +dev = [ + # fixed version so no new linter/formatter errors are introduced over time + # probably should not update outside the template + "ruff~=0.15.0", + "pytest~=9.0.2", + "basedpyright~=1.38.3", + "datamodel-code-generator>=0.26.5", - "mypy>=1.14.1", "pymdown-extensions>=10.14.1", - "ruff>=0.15.2", "setuptools>=75.8.0", "types-psycopg2>=2.9.21.20250121", "types-pyyaml>=6.0.12.20241230", "types-regex>=2024.11.6.20241221", "fabric>=3.2.2", -] - -[project.optional-dependencies] -test = [ "pandas-stubs>=2.3.2.250827", "parameterized>=0.9.0", - "pytest>=8.3.4", "testcontainers-postgres>=0.0.1rc1", ] @@ -97,20 +95,64 @@ select = [ "S101", ] +[tool.basedpyright] +typeCheckingMode = "off" +venvPath = "." +venv = ".venv" + +### warns +reportDeprecated = "warning" +reportIncompatibleMethodOverride = "warning" +reportIncompatibleVariableOverride = "warning" +reportMatchNotExhaustive = "warning" + +### pyright issues +reportGeneralTypeIssues = "error" +reportFunctionMemberAccess = "error" +reportMissingImports = "error" +reportMissingModuleSource = "error" +reportInvalidTypeForm = "error" +reportUnusedImport = "error" +reportUnusedClass = "error" +reportUnusedFunction = "error" +reportUnusedVariable = "error" +reportDuplicateImport = "error" +reportWildcardImportFromLibrary = "error" +reportAbstractUsage = "error" +reportAssertTypeFailure = "error" +reportCallIssue = "error" +reportInconsistentOverload = "error" +reportIndexIssue = "error" +reportInvalidTypeArguments = "error" +reportNoOverloadImplementation = "error" +reportRedeclaration = "error" +reportConstantRedefinition = "error" +reportPossiblyUnboundVariable = "error" +reportInvalidStringEscapeSequence = "error" +reportUnnecessaryIsInstance = "error" +reportUnnecessaryCast = "error" +reportUnnecessaryComparison = "error" +reportUnnecessaryContains = "error" +reportAssertAlwaysTrue = "error" +reportSelfClsParameterName = "error" +reportUndefinedVariable = "error" +reportUnhashable = "error" +reportUnsupportedDunderAll = "error" +reportUnusedCoroutine = "error" +reportUnusedExcept = "error" +reportUnusedExpression = "error" +reportUnnecessaryTypeIgnoreComment = "error" +reportUnreachable = "error" + +### based pyright issues +reportIgnoreCommentWithoutRule = "error" +reportImplicitRelativeImport = "error" +reportUnsafeMultipleInheritance = "error" +reportImplicitAbstractClass = "error" + + [tool.ruff.lint.per-file-ignores] "__init__.py" = [ "F403", # ignore star imports warning in __init__ files ] "tests/**" = ["S101"] - -[[tool.mypy.overrides]] -module = "astropy.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "testcontainers.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "astroquery.*" -ignore_missing_imports = true diff --git a/tests/integration/create_table_test.py b/tests/integration/create_table_test.py index acaedb44..e935f367 100644 --- a/tests/integration/create_table_test.py +++ b/tests/integration/create_table_test.py @@ -32,7 +32,7 @@ def test_create_table(self): presentation.CreateSourceRequest(title="title", authors=["author"], year=2022) ).code - response, created = self.upload_manager.create_table( + self.upload_manager.create_table( presentation.CreateTableRequest( table_name="table_name", columns=[ @@ -61,7 +61,7 @@ def test_create_table_with_patch(self): ).code table_name = "table_name" - response, created = self.upload_manager.create_table( + _, created = self.upload_manager.create_table( presentation.CreateTableRequest( table_name=table_name, columns=[ diff --git a/tests/integration/layer0_tables_repository_test.py b/tests/integration/layer0_tables_repository_test.py index a121cae8..1af9da00 100644 --- a/tests/integration/layer0_tables_repository_test.py +++ b/tests/integration/layer0_tables_repository_test.py @@ -44,5 +44,5 @@ def test_write_and_fetch_table(self): self.assertEqual(list(fetched_data.columns), ["ra", "dec"]) np.testing.assert_array_equal(fetched_data["ra"], test_data["ra"]) - self.assertEqual(fetched_data["ra"].unit, u.Unit("hour")) # type: ignore + self.assertEqual(fetched_data["ra"].unit, u.Unit("hour")) np.testing.assert_array_equal(fetched_data["dec"], test_data["dec"]) diff --git a/tests/integration/rawdata_table_test.py b/tests/integration/rawdata_table_test.py index aeb5561c..31eb783e 100644 --- a/tests/integration/rawdata_table_test.py +++ b/tests/integration/rawdata_table_test.py @@ -42,7 +42,7 @@ def test_create_table_happy_case(self): ], ) - table_resp, _ = self.manager.create_table( + self.manager.create_table( presentation.CreateTableRequest( table_name="test_table", columns=[ @@ -90,7 +90,7 @@ def test_create_table_with_nulls(self): ], ) - table_resp, _ = self.manager.create_table( + self.manager.create_table( presentation.CreateTableRequest( table_name="test_table", columns=[ @@ -171,7 +171,7 @@ def test_add_data_to_unknown_column(self): ], ) - table_resp, _ = self.manager.create_table( + self.manager.create_table( presentation.CreateTableRequest( table_name="test_table", columns=[ diff --git a/tests/regression/upload_simple_table.py b/tests/regression/upload_simple_table.py index 2b5c03cc..fd244159 100644 --- a/tests/regression/upload_simple_table.py +++ b/tests/regression/upload_simple_table.py @@ -105,7 +105,7 @@ def upload_data( request_data = adminapi.AddDataRequest( table_name=table_name, - data=df.to_dict("records"), # type: ignore + data=df.to_dict("records"), ) response = session.post("/v1/table/data", json=request_data.model_dump(mode="json")) diff --git a/tests/unit/data/layer0_repository_test.py b/tests/unit/data/layer0_repository_test.py index df320c0f..0bb2d939 100644 --- a/tests/unit/data/layer0_repository_test.py +++ b/tests/unit/data/layer0_repository_test.py @@ -51,10 +51,10 @@ def setUp(self) -> None: ), ] ) - def test_fetch_raw_data(self, name: str, kwargs: dict, expected_query: str): + def test_fetch_raw_data(self, _: str, kwargs: dict, expected_query: str): lib.returns(self.storage_mock.query, {"haha": [1, 2]}) - _ = self.repo.fetch_raw_data("ironman", **kwargs) + self.repo.fetch_raw_data("ironman", **kwargs) args, _ = self.storage_mock.query.call_args actual = normalize_query(args[0]) diff --git a/tests/unit/domain/table_upload_test.py b/tests/unit/domain/table_upload_test.py index 5c122977..df693bfd 100644 --- a/tests/unit/domain/table_upload_test.py +++ b/tests/unit/domain/table_upload_test.py @@ -120,7 +120,7 @@ def test_add_data_identical_rows(self): ) def test_create_table( self, - name: str, + _: str, request: presentation.CreateTableRequest, table_already_existed: bool = False, expected_created: bool = True, @@ -131,7 +131,7 @@ def test_create_table( if err_substr is not None: with self.assertRaises(errors.RuleValidationError) as err: - _, _ = self.manager.create_table(request) + self.manager.create_table(request) self.assertIn(err_substr, err.exception.message()) else: @@ -244,14 +244,14 @@ class TestData: ) def test_mapping( self, - name: str, + _: str, input_columns: list[presentation.ColumnDescription], expected: list[model.ColumnDescription] | None = None, err_substr: str | None = None, ): if err_substr: with self.assertRaises(errors.RuleValidationError) as err: - _ = domain_descriptions_to_data(input_columns) + domain_descriptions_to_data(input_columns) self.assertIn(err_substr, err.exception.message()) else: diff --git a/tests/unit/lib/astronomy_test.py b/tests/unit/lib/astronomy_test.py index 4f12f4dc..6a7aed2e 100644 --- a/tests/unit/lib/astronomy_test.py +++ b/tests/unit/lib/astronomy_test.py @@ -60,7 +60,7 @@ def setUp(self): ) def test_apex_velocity( self, - name: str, + _: str, vel: float, lon: float, lat: float, @@ -113,7 +113,7 @@ def test_apex_velocity( ) def test_apex_velocity_with_uncertainties( self, - name: str, + _: str, vel, lon, lat,