Resolution madness

.github/workflows/unittests.yml

@@ -0,0 +1,105 @@
+name: Unit tests
+
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+
+jobs:
+  format:
+    if: |
+      (github.event_name == 'push' || github.event.pull_request.head.repo.fork) &&
+      (!contains(github.event.head_commit.message, 'skip ci'))
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.13
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black toml
+      - name: Format source w/ black
+        # fail when file(s) would be formatted
+        run: black --check arrow_expts tests
+
+  lint:
+    if: |
+      (github.event_name == 'push' || github.event.pull_request.head.repo.fork) &&
+      (!contains(github.event.head_commit.message, 'skip ci'))
+
+    strategy:
+      matrix:
+        pyver: ["py310", "py313"]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Ruff with Python target version ${{ matrix.pyver }}
+        uses: chartboost/ruff-action@v1
+        with:
+          src: arrow_expts
+          args: "check --target-version ${{ matrix.pyver }}"
+
+  test:
+    if: |
+      (github.event_name == 'push' || github.event.pull_request.head.repo.fork) &&
+      (!contains(github.event.head_commit.message, 'skip ci'))
+    needs: [format, lint]
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.13"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        # include:
+        #   - python-version: "3.14"
+        #     os: ubuntu-latest
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          allow-prereleases: true
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+      - name: Run pytest
+        run: |
+          pytest -vv tests
+
+  type-hints:
+    if: |
+      (github.event_name == 'push' || github.event.pull_request.head.repo.fork) &&
+      (!contains(github.event.head_commit.message, 'skip ci'))
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.13", "3.14"]
+
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          allow-prereleases: true
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install mypy{,_extensions}
+      - name: Type check w/ mypy
+        run: echo y | mypy --install-types --check-untyped-defs --pretty arrow_expts

arrow_expts/schema/json-blob-schema.json

@@ -61,6 +61,20 @@
               },
               "type": "array"
             },
+            {
+              "items": {
+                "anyOf": [
+                  {
+                    "format": "binary",
+                    "type": "string"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "type": "array"
+            },
             {
               "items": {
                 "anyOf": [
@@ -112,6 +126,7 @@
             "integer",
             "number",
             "boolean",
+            "bytes",
             "date-time",
             "duration",
             "time-pattern"
@@ -280,6 +295,20 @@
               },
               "type": "array"
             },
+            {
+              "items": {
+                "anyOf": [
+                  {
+                    "format": "binary",
+                    "type": "string"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "type": "array"
+            },
             {
               "items": {
                 "anyOf": [
@@ -331,6 +360,7 @@
             "integer",
             "number",
             "boolean",
+            "bytes",
             "date-time",
             "duration",
             "time-pattern"
@@ -501,6 +531,20 @@
               },
               "type": "array"
             },
+            {
+              "items": {
+                "anyOf": [
+                  {
+                    "format": "binary",
+                    "type": "string"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "type": "array"
+            },
             {
               "items": {
                 "anyOf": [
@@ -552,6 +596,7 @@
             "integer",
             "number",
             "boolean",
+            "bytes",
             "date-time",
             "duration",
             "time-pattern"

arrow_expts/schema/models.py

@@ -2,65 +2,100 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#   "python-dateutil",
 #   "pandas>=2",
 #   "pydantic>=2",
 # ]
 # ///
 
-"""Write JSON schema for JSON blob in SpineDB
+"""Write JSON schema for JSON blob in SpineDB"""
 
-"""
-
-# from dataclasses import dataclass
-# from dataclasses import field
 from datetime import datetime, timedelta
-from typing import Annotated, Literal, Type, TypeAlias
+from typing import Annotated, Literal, TypeAlias
 
+from dateutil.relativedelta import relativedelta
+import numpy as np
 import pandas as pd
-from pydantic import RootModel
-from pydantic.dataclasses import dataclass
-from pydantic.dataclasses import Field as field
-from pydantic.types import StringConstraints
+
+if __name__ == "__main__":
+    from pydantic.dataclasses import dataclass
+    from pydantic.dataclasses import Field as field
+else:
+    from dataclasses import dataclass
+    from dataclasses import field
 
 
 Floats: TypeAlias = list[float]
 Integers: TypeAlias = list[int]
 Strings: TypeAlias = list[str]
 Booleans: TypeAlias = list[bool]
+BytesList: TypeAlias = list[bytes]  # array of bytes to support mixed types
 
 Datetimes: TypeAlias = list[datetime]
 Timedeltas: TypeAlias = list[timedelta]
 
 # FIXME: how to do w/o Pydantic?
 time_pat_re = r"(Y|M|D|WD|h|m|s)[0-9]+-[0-9]+"
-TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)]
+
+# generate schema w/ Pydantic by running as a script
+if __name__ == "__main__":
+    from pydantic.types import StringConstraints
+
+    TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)]
+else:
+    from re import Pattern
+
+    @dataclass(frozen=True)
+    class TimePattern:
+        pattern: str
+        re: str | Pattern[str] = time_pat_re
+
+
 TimePatterns: TypeAlias = list[TimePattern]
 
+# nullable variant of arrays
 NullableIntegers: TypeAlias = list[int | None]
 NullableFloats: TypeAlias = list[float | None]
 NullableStrings: TypeAlias = list[str | None]
 NullableBooleans: TypeAlias = list[bool | None]
+NullableBytesList: TypeAlias = list[bytes | None]
 NullableDatetimes: TypeAlias = list[datetime | None]
 NullableTimedeltas: TypeAlias = list[timedelta | None]
 NullableTimePatterns: TypeAlias = list[TimePattern | None]
 
+# sets of types used to define array schemas below
 IndexTypes: TypeAlias = Integers | Strings | Datetimes | Timedeltas | TimePatterns
 ValueTypes: TypeAlias = (
-    Integers | Strings | Floats | Booleans | Datetimes | Timedeltas | TimePatterns
+    Integers
+    | Strings
+    | Floats
+    | Booleans
+    | Datetimes
+    | Timedeltas
+    | TimePatterns
+    | BytesList
 )
 NullableValueTypes: TypeAlias = (
     NullableIntegers
     | NullableStrings
     | NullableFloats
     | NullableBooleans
+    | NullableBytesList
     | NullableDatetimes
     | NullableTimedeltas
     | NullableTimePatterns
 )
 
-
+# names of types used in the schema
 ValueTypeNames: TypeAlias = Literal[
-    "string", "integer", "number", "boolean", "date-time", "duration", "time-pattern"
+    "string",
+    "integer",
+    "number",
+    "boolean",
+    "bytes",
+    "date-time",
+    "duration",
+    "time-pattern",
 ]
 IndexValueTypeNames: TypeAlias = Literal[
     "string", "integer", "date-time", "duration", "time-pattern"
@@ -69,13 +104,25 @@
 type_map: dict[type, ValueTypeNames] = {
     str: "string",
     int: "integer",
+    np.int8: "integer",
+    np.int16: "integer",
+    np.int32: "integer",
+    np.int64: "integer",
     float: "number",
+    np.float16: "number",
+    np.float32: "number",
+    np.float64: "number",
+    # np.float128: "number",  # not available on macos
     bool: "boolean",
+    np.bool: "boolean",
     datetime: "date-time",
     pd.Timestamp: "date-time",
     timedelta: "duration",
     pd.Timedelta: "duration",
+    relativedelta: "duration",
+    pd.DateOffset: "duration",
     TimePattern: "time-pattern",
+    bytes: "string",
 }
 
 
@@ -184,7 +231,7 @@ class Array(_TypeInferMixin):
 
 # NOTE: To add run-length encoding to the schema, add it to the
 # following type union following which, we need to implement a
-# converter to an Arrow array type
+# converter to a compatible pyarrow array type
 Table: TypeAlias = list[
     RunEndIndex | DictEncodedIndex | ArrayIndex | RunEndArray | DictEncodedArray | Array
 ]
@@ -195,6 +242,8 @@ class Array(_TypeInferMixin):
     import json
     from pathlib import Path
 
+    from pydantic import RootModel
+
     parser = ArgumentParser(__doc__)
     parser.add_argument("json_file", help="Path of JSON schema file to write")
     opts = parser.parse_args()