Add SCORM→JSON→DOCX pipeline and reorganize format/pipeline modules

Dockerfile

@@ -132,7 +132,6 @@ COPY --from=builder /usr/src /antlr_build/
 COPY qcon qcon/
 COPY api api/
 COPY pandoc pandoc/
-COPY restapi restapi/
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 

api/apps.py

@@ -1,5 +1,6 @@
 from django.apps import AppConfig
 from django.conf import settings
+from django.db import connection
 import sys
 import logging
 logger = logging.getLogger(__name__)
@@ -18,6 +19,18 @@ def ready(self):
             else:
                 logger.info("qconapi has started")
 
+            # Ensure database connection is ready before accessing the database
+            # This prevents the RuntimeWarning about accessing database during app initialization
+            try:
+                connection.ensure_connection()
+            except Exception:
+                # Database not ready yet, skip initialization
+                return
+
+            # Skip database operations during migrations
+            if 'migrate' in sys.argv or 'makemigrations' in sys.argv:
+                return
+
             from django.contrib.auth.models import User        
             if not User.objects.filter(username=settings.ADMIN_USERNAME).exists():
                 User.objects.create_superuser(

api/formats/__init__.py

@@ -0,0 +1 @@
+# Package for supported content formats.

api/formats/docx/__init__.py

@@ -0,0 +1 @@
+# DOCX format handlers.

api/process/endanswers.py → api/formats/docx/endanswers.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 import xml.etree.ElementTree as ET
-from ..models import EndAnswer
+from ...models import EndAnswer
 import re
 
 def get_endanswers(questionlibrary):

api/process/extract_images.py → api/formats/docx/extract_images.py

@@ -1,5 +1,5 @@
 import re
-from ..models import Image
+from ...models import Image
 
 def extract_images(questionlibrary):
     try:

api/process/formatter.py → api/formats/docx/formatter.py

@@ -32,9 +32,32 @@ def run_formatter(questionlibrary):
     maincontenttitle = root.find('maincontent_title')
     logger.debug("checking maincontent title")
     if maincontenttitle is not None:
-        main_title = (maincontenttitle.text).strip()
-        if main_title:
-            questionlibrary.main_title = (trim_text(main_title)).lstrip('# ')
+        raw_main = (maincontenttitle.text or "").strip()
+        if raw_main:
+            # Use the first H1 line as the title; remaining lines become root-level text
+            main_lines = raw_main.splitlines()
+            title_index = None
+            for idx, line in enumerate(main_lines):
+                if line.lstrip().startswith('#'):
+                    title_index = idx
+                    break
+
+            if title_index is not None:
+                main_title = main_lines[title_index].strip()
+                main_title = (trim_text(main_title)).lstrip('# ').strip()
+                main_text_lines = main_lines[title_index + 1:]
+            else:
+                # Fallback: treat the first line as title if no H1 is found
+                main_title = (trim_text(main_lines[0])).lstrip('# ').strip()
+                main_text_lines = main_lines[1:]
+
+            main_text = "\n".join(main_text_lines).strip()
+
+            if main_title:
+                questionlibrary.main_title = main_title
+            if main_text:
+                # Preserve raw markdown for root-level text
+                questionlibrary.main_text = main_text
             questionlibrary.save()
 
 # ==================================== BODY

api/process/parser.py → api/formats/docx/parser.py

@@ -1,6 +1,6 @@
 import os
 import xml.etree.ElementTree as ET
-from ..models import EndAnswer, Section, Question
+from ...models import EndAnswer, Section, Question
 from django.conf import settings
 
 import logging

api/process/sectioner.py → api/formats/docx/sectioner.py

@@ -3,7 +3,7 @@
 import xml.etree.ElementTree as ET
 # from .process_helper import markdown_to_plain, trim_text, markdown_to_html
 from api.tasks import markdown_to_plain, trim_text, markdown_to_html
-from ..models import Section
+from ...models import Section
 
 import logging
 newlogger = logging.getLogger(__name__)
@@ -60,6 +60,9 @@ def run_sectioner(questionlibrary):
                 sectionobject.raw_content = maincontent.text
                 sectionobject.is_main_content = True
                 sectionobject.title = questionlibrary.main_title
+                if questionlibrary.main_text:
+                    sectionobject.text = markdown_to_html(questionlibrary.main_text)
+                    sectionobject.is_text_displayed = True
 
             sectiontext = section.find('sectiontext')
             if sectiontext is not None:

api/process/splitter.py → api/formats/docx/splitter.py

@@ -1,8 +1,8 @@
 import os
 import subprocess
 import xml.etree.ElementTree as ET
-from ..models import Section
-from ..models import Question
+from ...models import Section
+from ...models import Question
 # from .process_helper import trim_text
 from api.tasks import trim_text
 import logging

api/formats/scorm/__init__.py

@@ -0,0 +1 @@
+# SCORM format handlers.

api/formats/scorm/manifest.py

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import xml.etree.cElementTree as ET
+
+NS_D2L = "http://desire2learn.com/xsd/d2lcp_v2p0"
+NS_IMS = "http://www.imsglobal.org/xsd/imscp_v1p1"
+
+
+class ManifestEntity(object):
+    resources = []
+
+    def __init__(self):
+        del self.resources[:]
+
+    def add_resource(self, manifest_resource_entity):
+        self.resources.append(manifest_resource_entity)
+
+
+class ManifestResourceEntity(object):
+    def __init__(self, identifier, resource_type, material_type, href, title = '', link_target = ''):
+        self.identifier = identifier
+        self.resource_type = resource_type
+        self.material_type = material_type
+        self.href = href
+        self.title = title
+        self.link_target = link_target
+
+
+def build_manifest_tree(manifest_entity: ManifestEntity, identifier: str = "MANIFEST_1") -> ET.ElementTree:
+    """
+    Build an imsmanifest.xml tree using shared namespaces/constants.
+    """
+    root = ET.Element(
+        "manifest",
+        {"xmlns:d2l_2p0": NS_D2L, "xmlns": NS_IMS, "identifier": identifier},
+    )
+    resources_el = ET.SubElement(root, "resources")
+    for resource in manifest_entity.resources:
+        ET.SubElement(
+            resources_el,
+            "resource",
+            {
+                "identifier": resource.identifier,
+                "type": resource.resource_type,
+                "d2l_2p0:material_type": resource.material_type,
+                "href": resource.href,
+                "d2l_2p0:link_target": resource.link_target,
+                "title": resource.title,
+            },
+        )
+    return ET.ElementTree(root)
+
+
+def parse_manifest_tree(tree: ET.ElementTree) -> dict:
+    """
+    Parse an imsmanifest.xml ElementTree into a simple dict structure
+    consistent with XmlReader.parse_manifest output.
+    """
+    root = tree.getroot()
+    manifest_data = {
+        "identifier": root.get("identifier", ""),
+        "resources": [],
+    }
+    resources_el = root.find("resources")
+    if resources_el is not None:
+        for resource_el in resources_el.findall("resource"):
+            manifest_data["resources"].append(
+                {
+                    "identifier": resource_el.get("identifier", ""),
+                    "type": resource_el.get("type", ""),
+                    "material_type": resource_el.get(f"{{{NS_D2L}}}material_type", ""),
+                    "href": resource_el.get("href", ""),
+                    "link_target": resource_el.get(f"{{{NS_D2L}}}link_target", ""),
+                    "title": resource_el.get("title", ""),
+                }
+            )
+    return manifest_data

api/formats/scorm/manifest_builder.py

@@ -0,0 +1,29 @@
+import xml.etree.cElementTree as ET
+
+
+def build_manifest(manifest_entity):
+    root = ET.Element(
+        "manifest",
+        {
+            "xmlns:d2l_2p0": "http://desire2learn.com/xsd/d2lcp_v2p0",
+            "xmlns": "http://www.imsglobal.org/xsd/imscp_v1p1",
+            "identifier": "MANIFEST_1",
+        },
+    )
+    doc = ET.SubElement(root, "resources")
+
+    for resource in manifest_entity.resources:
+        ET.SubElement(
+            doc,
+            "resource",
+            {
+                "identifier": resource.identifier,
+                "type": resource.resource_type,
+                "d2l_2p0:material_type": resource.material_type,
+                "href": resource.href,
+                "d2l_2p0:link_target": resource.link_target,
+                "title": resource.title,
+            },
+        )
+
+    return ET.ElementTree(root)