rename uploading plugin & make better defaults for vizier task (#21)

* rename uploading plugin & make better defaults for vizier task

* log better

Author: Kraysent

app/__init__.py

@@ -2,14 +2,14 @@
     BibcodeProvider,
     DefaultTableNamer,
     DescriptionProvider,
-    UploaderPlugin,
+    UploaderSource,
 )
 from app.log import logger
 from app.tap import Constraint, TAPRepository
 from app.upload import upload
 
 __all__ = [
-    "UploaderPlugin",
+    "UploaderSource",
     "DefaultTableNamer",
     "BibcodeProvider",
     "DescriptionProvider",

app/interface.py

@@ -6,7 +6,7 @@
 from app.gen.client.adminapi import models
 
 
-class UploaderPlugin(abc.ABC):
+class UploaderSource(abc.ABC):
     @abc.abstractmethod
     def prepare(self) -> None:
         """

app/sources/csv.py

@@ -18,7 +18,7 @@
 
 
 @final
-class CSVSource(app.UploaderPlugin, app.DefaultTableNamer):
+class CSVSource(app.UploaderSource, app.DefaultTableNamer):
     def __init__(self, filename: str) -> None:
         self.filename = filename
         self._chunk_size = 1000

app/sources/fits.py

@@ -19,7 +19,7 @@
 
 
 @final
-class FITSSource(app.UploaderPlugin, app.DefaultTableNamer):
+class FITSSource(app.UploaderSource, app.DefaultTableNamer):
     def __init__(self, filename: str, hdu_index: int = 1) -> None:
         self.filename = filename
         self.hdu_index = hdu_index

app/sources/vizier.py

@@ -49,7 +49,7 @@ def _map_votable_datatype(datatype: str) -> models.DatatypeEnum:
 
 @final
 class VizierSource(
-    app.UploaderPlugin,
+    app.UploaderSource,
     app.DefaultTableNamer,
     app.BibcodeProvider,
     app.DescriptionProvider,

app/sources/vizier_v2.py

@@ -48,7 +48,7 @@ def dtype_to_datatype(dtype: str | np.dtype) -> models.DatatypeEnum:
 
 @final
 class VizierV2Source(
-    app.UploaderPlugin,
+    app.UploaderSource,
     app.DefaultTableNamer,
     app.BibcodeProvider,
     app.DescriptionProvider,

app/upload.py

@@ -18,7 +18,7 @@
 
 
 def upload(
-    plugin: interface.UploaderPlugin,
+    plugin: interface.UploaderSource,
     client: adminapi.AuthenticatedClient,
     *args,
     dry_run: bool = False,
@@ -54,7 +54,7 @@ def sanitize_value(val: Any) -> Any:
 
 
 def _upload(
-    plugin: interface.UploaderPlugin,
+    plugin: interface.UploaderSource,
     client: adminapi.AuthenticatedClient,
     table_name: str,
     table_description: str,
@@ -165,7 +165,7 @@ def process_chunk(data: Any) -> None:
 
 
 def upload_for_web(
-    plugin: interface.UploaderPlugin,
+    plugin: interface.UploaderSource,
     client: adminapi.AuthenticatedClient,
     table_name: str,
     table_description: str,

server/forms/upload_vizier.py

@@ -1,5 +1,6 @@
+import urllib.parse
 from collections.abc import Callable
-from typing import cast
+from typing import Literal, cast
 
 from pydantic import BaseModel, Field
 
@@ -8,14 +9,43 @@
 from app.gen.client import adminapi
 from app.sources.vizier import VizierSource
 from app.upload import upload_for_web
-from server.forms.upload_base import UploadBaseForm
 
 
-class UploadVizierForm(UploadBaseForm):
+class UploadVizierForm(BaseModel):
     catalog_name: str = Field(..., title="VizieR catalog name")
     source_table_name: str = Field(..., title="VizieR table name")
     cache_path: str = Field(default=".vizier_cache/", title="Cache path")
     batch_size: int = Field(default=100, title="Batch size", ge=1)
+    table_name: str = Field(
+        default="",
+        title="Table name",
+        description="Machine-readable HyperLEDA table id; leave empty to derive from VizieR.",
+    )
+    table_description: str = Field(
+        default="",
+        title="Table description",
+        description="Human-readable description; leave empty to take from VizieR metadata.",
+    )
+    bibcode: str = Field(
+        default="",
+        title="Bibcode",
+        description="NASA ADS bibcode; leave empty to read from VizieR.",
+    )
+    table_type: Literal["regular", "compilation"] = Field(
+        default="regular",
+        title="Table type",
+        description="Type of data that table represents.",
+    )
+    dry_run: bool = Field(
+        default=False,
+        title="Dry run",
+        description="Show schema and process rows without actually uploading any data. Useful for validation.",
+    )
+    endpoint: Literal["dev", "test", "prod"] = Field(
+        default="prod",
+        title="API endpoint",
+        description="Where to upload. Leave unchanged if https://leda.sao.ru is needed.",
+    )
 
 
 def handle_upload_vizier(form: BaseModel, report_func: Callable[[report.Event], None]) -> None:
@@ -24,22 +54,55 @@ def handle_upload_vizier(form: BaseModel, report_func: Callable[[report.Event],
         base_url=env_map[f.endpoint],
         token="fake",
     )
-    source = VizierSource(f.catalog_name, f.source_table_name, cache_path=f.cache_path, batch_size=f.batch_size)
-    bibcode = f.bibcode.strip() if f.has_bibcode else ""
-    pub_name = f.pub_name.strip()
-    pub_authors = list(f.pub_authors)
-    pub_year = f.pub_year
+    source = VizierSource(
+        f.catalog_name,
+        f.source_table_name,
+        cache_path=f.cache_path,
+        batch_size=f.batch_size,
+    )
+
+    table_name_in = f.table_name.strip()
+    if table_name_in:
+        resolved_table_name = table_name_in
+    else:
+        resolved_table_name = source.get_table_name()
+
+    table_description_in = f.table_description.strip()
+    if table_description_in:
+        resolved_description = table_description_in
+    else:
+        resolved_description = source.get_description()
+
+    bibcode_in = f.bibcode.strip()
+    if bibcode_in:
+        resolved_bibcode = bibcode_in
+    else:
+        resolved_bibcode = source.get_bibcode()
+
+    if not resolved_bibcode:
+        msg = "bibcode is empty: provide one or ensure VizieR metadata includes it"
+        raise ValueError(msg)
+
+    report_func(report.LogEvent(message=f"Table name: {resolved_table_name}"))
+    report_func(report.LogEvent(message=f"Description: {resolved_description}"))
+    ads_url = "https://ui.adsabs.harvard.edu/abs/" + urllib.parse.quote(resolved_bibcode, safe="")
+    report_func(
+        report.LogEvent(
+            message=f"Paper: {ads_url}",
+        ),
+    )
+
     table_type = (f.table_type or "regular").upper()
 
     upload_for_web(
         source,
         client,
-        f.table_name.strip(),
-        f.table_description.strip(),
-        bibcode,
-        pub_name,
-        pub_authors,
-        pub_year,
+        resolved_table_name,
+        resolved_description,
+        resolved_bibcode,
+        "",
+        [],
+        0,
         table_type,
         dry_run=f.dry_run,
         report_func=report_func,

tests/test_upload.py

@@ -8,12 +8,12 @@
 
 from app.gen.client import adminapi
 from app.gen.client.adminapi import models, types
-from app.interface import UploaderPlugin
+from app.interface import UploaderSource
 from app.sources.csv import CSVSource
 from app.upload import upload
 
 
-class StubPlugin(UploaderPlugin):
+class StubPlugin(UploaderSource):
     def __init__(self, should_raise: bool = False):
         self.should_raise = should_raise
         self.stop_called = False