SUBMIT-760: Entity - Geospatial File preview

submit-api/migrations/versions/4eab27236b2a_add_geo_data_uploads_table.py

@@ -0,0 +1,49 @@
+"""Add geo_data_uploads table
+
+Revision ID: 4eab27236b2a
+Revises: 77862b346adf
+Create Date: 2026-03-19 11:37:00.995726
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4eab27236b2a'
+down_revision = '77862b346adf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('geo_data_uploads',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('filename', sa.String(length=255), nullable=False),
+    sa.Column('file_type', sa.String(length=10), nullable=True),
+    sa.Column('file_size_kb', sa.Float(), nullable=True),
+    sa.Column('raw_s3_key', sa.String(length=512), nullable=True),
+    sa.Column('preview_s3_key', sa.String(length=512), nullable=True),
+    sa.Column('standard_s3_key', sa.String(length=512), nullable=True),
+    sa.Column('status', sa.String(length=20), nullable=False),
+    sa.Column('error_message', sa.Text(), nullable=True),
+    sa.Column('feature_count', sa.Integer(), nullable=True),
+    sa.Column('geometry_type', sa.String(length=50), nullable=True),
+    sa.Column('crs_original', sa.String(length=100), nullable=True),
+    sa.Column('bbox', sa.JSON(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.Column('created_date', sa.DateTime(), nullable=False),
+    sa.Column('updated_date', sa.DateTime(), nullable=True),
+    sa.Column('created_by', sa.String(length=50), nullable=True),
+    sa.Column('updated_by', sa.String(length=50), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('geo_data_uploads')
+    # ### end Alembic commands ###

submit-api/requirements/prod.txt

@@ -24,4 +24,8 @@ python-dotenv
 requests
 flask_cors
 pyhumps
-importlib-resources
+importlib-resources
+geopandas
+fiona
+pyproj
+shapely

submit-api/src/submit_api/__init__.py

@@ -44,7 +44,7 @@
 def create_app(run_mode=os.getenv('FLASK_ENV', 'development')):
     """Create flask app."""
     from submit_api.resources import (  # pylint: disable=import-outside-toplevel
-        API_BLUEPRINT, OPS_BLUEPRINT, STAFF_API_BLUEPRINT)
+        API_BLUEPRINT, GEO_API_BLUEPRINT, OPS_BLUEPRINT, STAFF_API_BLUEPRINT)
 
     # Flask app initialize
     app = Flask(__name__)
@@ -54,7 +54,7 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'development')):
 
     CORS(app, origins=allowedorigins(), supports_credentials=True)
 
-    # Register blueprints
+    app.register_blueprint(GEO_API_BLUEPRINT)
     app.register_blueprint(API_BLUEPRINT)
     app.register_blueprint(OPS_BLUEPRINT)
     app.register_blueprint(STAFF_API_BLUEPRINT)

submit-api/src/submit_api/config.py

@@ -92,6 +92,8 @@ class _Config():  # pylint: disable=too-few-public-methods
     KEYCLOAK_ADMIN_SECRET = _get_config("KEYCLOAK_ADMIN_SECRET")
     CONNECT_TIMEOUT = _get_config("CONNECT_TIMEOUT", default=60)
 
+    DOCUMENT_SERVICE_URL = _get_config("DOCUMENT_SERVICE_URL")
+
     BASE_APP_URL = os.getenv('BASE_APP_URL')
     SIGNUP_URL_PATH = os.getenv('SIGNUP_URL_PATH')
 

submit-api/src/submit_api/models/__init__.py

@@ -48,3 +48,4 @@
 from .track_phase import TrackPhase
 from .track_work import TrackWork
 from .account_project_work import AccountProjectWork
+from .geo_data_upload import GeoDataUpload

submit-api/src/submit_api/models/geo_data_upload.py

@@ -0,0 +1,66 @@
+# Copyright © 2024 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GeoDataUpload model class.
+
+Manages the geo data upload entity.
+"""
+from __future__ import annotations
+
+from datetime import datetime, UTC
+
+from sqlalchemy import Column, DateTime, Float, Integer, JSON, String, Text
+
+from .base_model import BaseModel
+
+
+class GeoDataUpload(BaseModel):
+    """Definition of the GeoDataUpload entity."""
+
+    __tablename__ = 'geo_data_uploads'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    filename = Column(String(255), nullable=False)
+    file_type = Column(String(10), nullable=True)  # 'shp' or 'zip'
+    file_size_kb = Column(Float, nullable=True)
+    raw_s3_key = Column(String(512), nullable=True)
+    preview_s3_key = Column(String(512), nullable=True)
+    standard_s3_key = Column(String(512), nullable=True)
+    status = Column(String(20), nullable=False, default='processing')
+    error_message = Column(Text, nullable=True)
+    feature_count = Column(Integer, nullable=True)
+    geometry_type = Column(String(50), nullable=True)
+    crs_original = Column(String(100), nullable=True)
+    bbox = Column(JSON, nullable=True)
+    created_at = Column(DateTime, default=lambda: datetime.now(UTC), nullable=False)
+    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), nullable=True)
+
+    def to_dict(self):
+        """Return a dictionary representation of the model."""
+        return {
+            'id': self.id,
+            'filename': self.filename,
+            'file_type': self.file_type,
+            'file_size_kb': self.file_size_kb,
+            'raw_s3_key': self.raw_s3_key,
+            'preview_s3_key': self.preview_s3_key,
+            'standard_s3_key': self.standard_s3_key,
+            'status': self.status,
+            'error_message': self.error_message,
+            'feature_count': self.feature_count,
+            'geometry_type': self.geometry_type,
+            'crs_original': self.crs_original,
+            'bbox': self.bbox,
+            'created_at': self.created_at.isoformat() if self.created_at else None,
+            'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+        }

submit-api/src/submit_api/resources/__init__.py

@@ -46,14 +46,16 @@
 from .staff.activity_log import API as STAFF_ACTIVITY_LOG_API
 from .staff.submitted_document import API as SUBMITTED_DOCUMENT_API
 from .proponent.proponent import API as PROPONENT_API
+from .proponent.geo import API as GEO_UPLOAD_API
 from .staff.submission import API as STAFF_SUBMISSION_API
 
-__all__ = ('API_BLUEPRINT', 'OPS_BLUEPRINT', 'STAFF_API_BLUEPRINT')
+__all__ = ('API_BLUEPRINT', 'OPS_BLUEPRINT', 'STAFF_API_BLUEPRINT', 'GEO_API_BLUEPRINT')
 
 URL_PREFIX = '/api'
 API_BLUEPRINT = Blueprint('API', __name__, url_prefix=f"{URL_PREFIX}/")
 OPS_BLUEPRINT = Blueprint("API_OPS", __name__, url_prefix="/ops")
 STAFF_API_BLUEPRINT = Blueprint("STAFF_API", __name__, url_prefix=f"{URL_PREFIX}/staff")
+GEO_API_BLUEPRINT = Blueprint("GEO_API", __name__, url_prefix=f"{URL_PREFIX}/geo")
 
 API_OPS = Api(
     OPS_BLUEPRINT,
@@ -104,3 +106,12 @@
 STAFF_API.add_namespace(SUBMITTED_DOCUMENT_API)
 STAFF_API.add_namespace(PROPONENT_API)
 STAFF_API.add_namespace(STAFF_SUBMISSION_API)
+
+GEO_API = Api(
+    GEO_API_BLUEPRINT,
+    title='GEO SUBMIT API',
+    version='1.0',
+    description='The Core API for Geospatial Uploads'
+)
+
+GEO_API.add_namespace(GEO_UPLOAD_API, path="/uploads")

submit-api/src/submit_api/resources/proponent/geo.py

@@ -0,0 +1,163 @@
+# Copyright © 2024 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""API blueprint for geospatial upload management.
+
+All business logic lives in :mod:`submit_api.services.geo_processor`.
+These route handlers are intentionally thin: validate → delegate → respond.
+"""
+import json
+import logging
+import time
+from http import HTTPStatus
+
+from flask import Response, current_app, request
+from flask_cors import cross_origin
+from flask_restx import Namespace, Resource
+
+from submit_api.auth import auth
+from submit_api.models import GeoDataUpload, db
+from submit_api.services.geo_processor import GeoService
+from submit_api.utils.util import allowedorigins, cors_preflight
+
+API = Namespace("uploads", description="Endpoints for Geospatial Uploads")
+logger = logging.getLogger(__name__)
+
+
+@cors_preflight("GET, POST, OPTIONS")
+@API.route("", methods=["GET", "POST", "OPTIONS"])
+class GeoUploads(Resource):
+    """Resource for managing geo uploads."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def post():
+        """Create a GeoDataUpload record and kick off background processing."""
+        data = request.get_json()
+        try:
+            upload = GeoService.create_upload(
+                app=current_app._get_current_object(),  # noqa: SLF001
+                filename=data.get("filename"),
+                file_type=data.get("file_type"),
+                file_size_kb=data.get("file_size_kb"),
+                s3_key=data.get("s3_key"),
+            )
+        except ValueError as exc:
+            return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
+
+        return {"id": upload.id, "status": upload.status}, HTTPStatus.CREATED
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def get():
+        """Return the GeoDataUpload records for the specified item."""
+        item_id = request.args.get("item_id", type=int)
+        uploads = GeoService.list_uploads(item_id=item_id)
+        return [u.to_dict() for u in uploads], HTTPStatus.OK
+
+
+@cors_preflight("GET, OPTIONS")
+@API.route("/<int:upload_id>/status", methods=["GET", "OPTIONS"])
+class GeoUploadStatus(Resource):
+    """Resource for polling geo upload status."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def get(upload_id):
+        """SSE stream that polls the processing status of a single upload."""
+        app = current_app._get_current_object()  # noqa: SLF001
+
+        def generate():
+            with app.app_context():
+                for _ in range(60):
+                    upload = db.session.get(GeoDataUpload, upload_id)
+                    if not upload:
+                        yield 'data: {"error": "Upload not found"}\n\n'
+                        return
+
+                    payload = {"status": upload.status, "upload_id": upload.id}
+
+                    if upload.status == "ready":
+                        payload.update({
+                            "feature_count": upload.feature_count,
+                            "geometry_type": upload.geometry_type,
+                            "crs_original": upload.crs_original,
+                            "bbox": upload.bbox,
+                        })
+                        yield f"data: {json.dumps(payload)}\n\n"
+                        return
+
+                    if upload.status == "failed":
+                        payload["error"] = upload.error_message
+                        yield f"data: {json.dumps(payload)}\n\n"
+                        return
+
+                    yield f"data: {json.dumps(payload)}\n\n"
+                    time.sleep(1.5)
+                    db.session.expire_all()
+
+        return Response(
+            generate(),
+            mimetype="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+
+@cors_preflight("GET, OPTIONS")
+@API.route("/<int:upload_id>/url", methods=["GET", "OPTIONS"])
+class GeoUploadUrl(Resource):
+    """Resource for getting a presigned S3 read URL."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def get(upload_id):
+        """Return a fresh presigned S3 read URL for a processed GeoDataUpload."""
+        tier = request.args.get("tier", "preview")
+        try:
+            result = GeoService.get_presigned_read_url(upload_id, tier)
+        except LookupError as exc:
+            return {"error": str(exc)}, HTTPStatus.NOT_FOUND
+        except ValueError as exc:
+            return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
+        except Exception as exc:  # pylint: disable=broad-except # noqa: B902
+            logger.exception("Unexpected error fetching presigned URL: %s", exc)
+            return {"error": str(exc)}, HTTPStatus.INTERNAL_SERVER_ERROR
+
+        return result, HTTPStatus.OK
+
+
+@cors_preflight("POST, OPTIONS")
+@API.route("/<int:upload_id>/retry", methods=["POST", "OPTIONS"])
+class GeoUploadRetry(Resource):
+    """Resource for retrying an upload."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def post(upload_id):
+        """Re-trigger background processing for a failed upload."""
+        try:
+            upload = GeoService.retry_upload(
+                app=current_app._get_current_object(),  # noqa: SLF001
+                upload_id=upload_id,
+            )
+        except LookupError as exc:
+            return {"error": str(exc)}, HTTPStatus.NOT_FOUND
+        except ValueError as exc:
+            return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
+
+        return {"id": upload.id, "status": upload.status}, HTTPStatus.OK

submit-api/src/submit_api/resources/proponent/submission.py

@@ -15,6 +15,7 @@
 
 from http import HTTPStatus
 
+from flask import current_app, jsonify
 from flask_cors import cross_origin
 from flask_restx import Namespace, Resource
 
@@ -153,3 +154,27 @@ def post(submission_id):
         move_submission_data = CreateSubmissionRequestSchema().load(API.payload)
         created_submission = SubmissionService.move_submission(submission_id, move_submission_data)
         return SubmissionSchema().dump(created_submission), HTTPStatus.CREATED
+
+
+@cors_preflight("OPTIONS, POST")
+@API.route("/items/<int:submission_item_id>/geo-process", methods=["POST", "OPTIONS"])
+class SubmissionItemGeoProcess(Resource):
+    """Resource to trigger geospatial processing for a submission item."""
+
+    @staticmethod
+    @ApiHelper.swagger_decorators(API, endpoint_description="Trigger geospatial processing")
+    @API.response(HTTPStatus.ACCEPTED, "Processing triggered")
+    @API.response(HTTPStatus.BAD_REQUEST, "Bad Request")
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def post(submission_item_id):
+        """Trigger geospatial processing for newly uploaded files."""
+        try:
+            triggered_uploads = SubmissionService.trigger_geo_process(submission_item_id)
+            return jsonify({
+                'message': f'Triggered {len(triggered_uploads)} geo-processes',
+                'uploads': triggered_uploads
+            }), HTTPStatus.ACCEPTED
+        except Exception as e:  # noqa: B902
+            current_app.logger.exception(f"Error triggering geo process: {e}")
+            return jsonify({'error': str(e)}), HTTPStatus.INTERNAL_SERVER_ERROR