Backend libs update

server/Pipfile

@@ -4,25 +4,25 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-connexion = {extras = ["swagger-ui"],version = "==2.14.1"}
-flask = "==2.2.5"
+connexion = {extras = ["swagger-ui"],version = "==2.15.1"}
+flask = "==3.1.2"
 python-dateutil = "==2.8.2"
-marshmallow = "==3.20.1"
-flask-marshmallow = "==0.14.0"
-marshmallow-sqlalchemy = "==1.1.0"
+marshmallow = "==3.26.1"
+flask-marshmallow = "==0.15.0"
+marshmallow-sqlalchemy = "==1.4.1"
 psycopg2-binary = "==2.9.9"
 itsdangerous = "==2.2.0"
-Flask-SQLAlchemy = "==2.5.1"
-sqlalchemy = "==1.4.53"
-gunicorn = {extras = ["gevent"],version = "==19.9"}
+Flask-SQLAlchemy = "==3.1.1"
+sqlalchemy = "==2.0.44"
+gunicorn = {extras = ["gevent"],version = "==23.0"}
 python-dotenv = "==0.20.0"
-flask-login = "==0.6.2"
+flask-login = "==0.6.3"
 bcrypt = "==4.2.0"
-wtforms = {extras = ["email"],version = "==3.1.2"}
-flask-wtf = "==1.0.1"
+wtforms = {extras = ["email"],version = "==3.2.1"}
+flask-wtf = "==1.2.2"
 flask-mail = "==0.10.0"
 safe = "==0.4"
-flask-migrate = "==2.6.0" # 3.1.0
+flask-migrate = "==3.1.0"
 wtforms-json = "==0.3.5"
 pytz = "==2022.2.1"
 scikit-build = "==0.18.1"

server/mergin/app.py

@@ -10,6 +10,7 @@
 import gevent
 from marshmallow import fields
 from sqlalchemy.schema import MetaData
+from sqlalchemy import text
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask import (
@@ -27,7 +28,6 @@
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
 from flask_mail import Mail
-from connexion.apps.flask_app import FlaskJSONEncoder
 from flask_wtf import FlaskForm
 from wtforms import StringField
 from pathlib import Path
@@ -37,7 +37,6 @@
 from werkzeug.exceptions import HTTPException
 from typing import List, Dict, Optional, Tuple
 
-from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
 from .config import Configuration
 from .commands import add_commands as server_commands
 
@@ -139,7 +138,6 @@ def create_simple_app() -> Flask:
     app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir))
     flask_app = app.app
 
-    flask_app.json_encoder = FlaskJSONEncoder
     flask_app.config.from_object(Configuration)
     db.init_app(flask_app)
     ma.init_app(flask_app)
@@ -155,54 +153,36 @@ def create_simple_app() -> Flask:
 def create_app(public_keys: List[str] = None) -> Flask:
     """Factory function to create Flask app instance"""
     from itsdangerous import BadTimeSignature, BadSignature
-    from .auth import auth_required, decode_token
-    from .auth.models import User
 
-    # from .celery import celery
-    from .sync.db_events import register_events
-    from .sync.workspace import GlobalWorkspaceHandler
-    from .sync.config import Configuration as SyncConfig
-    from .sync.commands import add_commands
-    from .auth import register as register_auth
+    from .auth import auth_required, decode_token, register as register_auth
+    from .auth.models import User
+    from .sync.app import register as register_sync
     from .sync.project_handler import ProjectHandler
+    from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
+    from .sync.workspace import GlobalWorkspaceHandler
 
     app = create_simple_app().connexion_app
 
-    app.add_api(
-        "sync/public_api.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/public_api_v2.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/private_api.yaml",
-        base_path="/app",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": False, "serve_spec": False},
-        validate_responses=True,
-    )
     app.add_api(
         "api.yaml",
         arguments={"title": "Mergin"},
         options={"swagger_ui": False, "serve_spec": False},
         validate_responses=True,
     )
+    app.app.blueprints["/"].name = "main"
+    app.app.blueprints["main"] = app.app.blueprints.pop("/")
 
-    app.app.config.from_object(SyncConfig)
-    app.app.connexion_app = app
+    # register sync module
+    register_sync(app.app)
 
+    # initialize extensions
     mail.init_app(app.app)
-    app.mail = mail
     csrf.init_app(app.app)
     login_manager.init_app(app.app)
+
     # register auth blueprint
     register_auth(app.app)
+
     server_commands(app.app)
 
     # adjust login manager
@@ -228,8 +208,6 @@ def load_user_from_header(header_val):  # pylint: disable=W0613,W0612
             except (BadSignature, BadTimeSignature, KeyError):
                 pass
 
-    # csrf = app.app.extensions['csrf']
-
     @app.app.before_request
     def check_maintenance():
         allowed_endpoints = ["/project/by_names", "/auth/login", "/alive"]
@@ -275,9 +253,6 @@ def get_startup_data():
         }
         return data
 
-    # update celery config with flask app config
-    # celery.conf.update(app.app.config)
-
     @app.route("/alive", methods=["POST"])
     @csrf.exempt
     def alive():  # pylint: disable=E0722
@@ -287,7 +262,7 @@ def alive():  # pylint: disable=E0722
         start_time = time.time()
         try:
             with db.engine.connect() as con:
-                rs = con.execute("SELECT 2 * 2")
+                rs = con.execute(text("SELECT 2 * 2"))
                 assert rs.fetchone()[0] == 4
         except:
             """Although bad form, we have deliberate left this except broad. When we have an uncaught exception in
@@ -388,7 +363,6 @@ def init():  # pylint: disable=W0612
         response.headers.set("X-CSRF-Token", generate_csrf())
         return response
 
-    register_events()
     application = app.app
 
     @application.errorhandler(Exception)
@@ -467,8 +441,6 @@ def config():
             cfg["build_hash"] = application.config["BUILD_HASH"]
             return jsonify(cfg), 200
 
-    # append project commands (from default sync module)
-    add_commands(application)
     return application
 
 

server/mergin/auth/controller.py

@@ -396,7 +396,7 @@ def register_user():  # pylint: disable=W0613,W0612
 
     form = UserRegistrationForm()
     form.username.data = User.generate_username(form.email.data)
-    if form.validate_on_submit():
+    if form.is_submitted() and form.validate():
         user = User.create(form.username.data, form.email.data, form.password.data)
         user_created.send(user, source="admin")
         token = generate_confirmation_token(
@@ -497,8 +497,9 @@ def get_paginated_users(
     elif not descending and order_by:
         users = users.order_by(asc(User.__table__.c[order_by]))
 
-    result = users.paginate(page, per_page).items
-    total = users.paginate(page, per_page).total
+    paginate = users.paginate(page=page, per_page=per_page)
+    result = paginate.items
+    total = paginate.total
 
     result_users = UserSchema(many=True).dump(result)
 

server/mergin/auth/tasks.py

@@ -14,7 +14,7 @@
 @celery.task
 def anonymize_removed_users():
     """Permanently 'delete' users marked for removal by removing personal information"""
-    db.session.info = {"msg": "anonymize_removed_users"}
+    db.session.info["msg"] = "anonymize_removed_users"
     before_expiration = datetime.today() - timedelta(Configuration.ACCOUNT_EXPIRATION)
     users = User.query.filter(
         isnot(User.active, True),

server/mergin/commands.py

@@ -7,6 +7,7 @@
 import random
 import string
 import os
+from sqlalchemy import inspect
 
 
 def _echo_title(title):
@@ -136,7 +137,8 @@ def _check_server(app: Flask):  # pylint: disable=W0612
     else:
         _echo_error("No service ID set.")
 
-    tables = db.engine.table_names()
+    inspect_engine = inspect(db.engine)
+    tables = inspect_engine.get_table_names()
     if not tables:
         _echo_error("Database not initialized. Run flask init-db command")
     else:
@@ -157,9 +159,9 @@ def _init_db(app: Flask):
         label="Creating database", length=4, show_eta=False
     ) as progress_bar:
         progress_bar.update(0)
-        db.drop_all(bind=None)
+        db.drop_all(bind_key=None)
         progress_bar.update(1)
-        db.create_all(bind=None)
+        db.create_all(bind_key=None)
         progress_bar.update(2)
         db.session.commit()
         progress_bar.update(3)
@@ -202,7 +204,8 @@ def init(email: str, recreate: bool):
         """Initialize database if does not exist or -r is provided. Perform check of server configuration. Send statistics, respecting your setup."""
         from .auth.models import User, UserProfile
 
-        tables = db.engine.table_names()
+        inspect_engine = inspect(db.engine)
+        tables = inspect_engine.get_table_names()
         if recreate and tables:
             click.confirm(
                 "Are you sure you want to recreate database and admin user? This will remove all data!",

server/mergin/controller.py

@@ -1,15 +1,14 @@
-import json
-import logging
-import os
+# Copyright (C) Lutra Consulting Limited
+#
+# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
 from flask import abort, current_app, request
 from flask_login import current_user
 from magic import from_buffer
-import time
-
 import requests
 
 from .utils import save_diagnostic_log_file
-from .app import parse_version_string, db
+from .app import parse_version_string
 
 
 def get_latest_version():

server/mergin/stats/models.py

@@ -2,11 +2,12 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
+import uuid
 from dataclasses import dataclass
 from typing import Optional
-import uuid
-from sqlalchemy.dialects.postgresql import UUID, JSONB
-from datetime import datetime, timezone
+from datetime import datetime
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
 
 from ..app import db
 
@@ -30,8 +31,8 @@ class ServerCallhomeData:
 class MerginInfo(db.Model):
     """Information about deployment"""
 
-    service_id = db.Column(UUID(as_uuid=True), primary_key=True)
-    last_reported = db.Column(db.DateTime)
+    service_id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
+    last_reported: Mapped[Optional[datetime]]
 
     def __init__(self, service_id: str = None):
         if service_id:
@@ -43,9 +44,7 @@ def __init__(self, service_id: str = None):
 class MerginStatistics(db.Model):
     """Information about deployment"""
 
-    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    created_at = db.Column(
-        db.DateTime, index=True, nullable=False, default=datetime.utcnow
-    )
+    id: Mapped[int] = mapped_column(primary_key=True)
+    created_at: Mapped[datetime] = mapped_column(index=True, default=datetime.utcnow)
     # data with statistics
-    data = db.Column(JSONB, nullable=False)
+    data: Mapped[dict] = mapped_column(JSONB)

server/mergin/stats/tasks.py

@@ -4,11 +4,13 @@
 
 from dataclasses import asdict
 import requests
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 import json
 import logging
 from flask import current_app
 from sqlalchemy.sql.operators import is_
+from sqlalchemy import inspect
+from sqlalchemy.sql import select, func
 
 from .models import MerginInfo, MerginStatistics, ServerCallhomeData
 from ..celery import celery
@@ -21,21 +23,25 @@ def get_callhome_data(info: MerginInfo | None = None) -> ServerCallhomeData:
     """
     Get data about server to send to callhome service
     """
-    last_change_item = (
-        db.session.query(Project.updated).order_by(Project.updated.desc()).first()
-    )
+    last_change = db.session.scalars(
+        select(Project.updated).order_by(Project.updated.desc())
+    ).first()
     service_uuid = str(info.service_id) if info else None
     data = ServerCallhomeData(
         service_uuid=service_uuid,
         url=current_app.config["MERGIN_BASE_URL"],
         contact_email=current_app.config["CONTACT_EMAIL"],
         licence=current_app.config["SERVER_TYPE"],
-        projects_count=Project.query.filter(Project.removed_at.is_(None)).count(),
-        users_count=User.query.filter(
-            is_(User.username.ilike("deleted_%"), False)
-        ).count(),
+        projects_count=db.session.scalar(
+            select(func.count(Project.id)).where(is_(Project.removed_at, None))
+        ),
+        users_count=db.session.scalar(
+            select(func.count(User.id)).where(
+                is_(User.username.ilike("deleted_%"), False)
+            )
+        ),
         workspaces_count=current_app.ws_handler.workspace_count(),
-        last_change=str(last_change_item.updated) + "Z" if last_change_item else "",
+        last_change=str(last_change) + "Z" if last_change else "",
         server_version=current_app.config["VERSION"],
         monthly_contributors=current_app.ws_handler.monthly_contributors_count(),
         editors=current_app.ws_handler.server_editors_count(),
@@ -47,7 +53,7 @@ def get_callhome_data(info: MerginInfo | None = None) -> ServerCallhomeData:
 @celery.task(ignore_result=True)
 def save_statistics():
     """Save statistics about usage."""
-    info = MerginInfo.query.first()
+    info = db.session.execute(select(MerginInfo)).scalar_one_or_none()
     data = get_callhome_data(info)
     stat = MerginStatistics(data=data)
     db.session.add(stat)
@@ -71,11 +77,12 @@ def send_statistics():
     if not current_app.config["COLLECT_STATISTICS"]:
         return
 
-    if not db.engine.has_table("mergin_info"):
+    inspect_engine = inspect(db.engine)
+    if not inspect_engine.has_table("mergin_info"):
         logging.warning("Database not initialized")
         return
 
-    info = MerginInfo.query.first()
+    info = db.session.execute(select(MerginInfo)).scalar_one_or_none()
     if not info:
         # create new info with random service id
         service_id = current_app.config.get("SERVICE_ID", None)