From 2c94542c0280b0a639a29a36096b560c463eab40 Mon Sep 17 00:00:00 2001 From: Ole Mussmann Date: Wed, 29 Jan 2025 18:32:32 +0100 Subject: [PATCH 01/43] low_res_datetime --- arrow_expts/spine/dbmap.py | 72 ++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 7b9e163..1002055 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -7,6 +7,7 @@ import weakref import pandas as pd +import numpy as np from spinedb_api import DatabaseMapping from spinedb_api.temp_id import TempId @@ -25,31 +26,64 @@ def filter_frequencies(freq: str) -> str: .replace("year", "Y") \ .replace("months", "M") \ .replace("month", "M") \ - .replace("quarters", "Q") \ - .replace("quarter", "Q") \ .replace("weeks", "W") \ .replace("week", "W") \ + .replace("days", "D") \ + .replace("day", "D") \ .replace("hours", "h") \ .replace("hour", "h") \ .replace("minutes", "min") \ .replace("minute", "min") \ .replace("seconds", "s") \ - .replace("second", "s") \ - .replace("microseconds", "us") \ - .replace("microsecond", "us") \ - .replace("nanoseconds", "ns") \ - .replace("nanosecond", "ns") + .replace("second", "s") if re.compile("^[0-9]+$").match(filtered_freq): # If frequency is an integer, the implied unit is "minutes" - return freq + "m" + return filtered_freq + "m" else: - return freq + return filtered_freq class IndexType(Enum): Timestamp = auto() Sequence = auto() Generic = auto() +to_numpy = { + "Y": "Y", + "M": "M", + "W": "W", + "D": "D", + "h": "h", + "min": "m", + "s": "s", +} + +def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: + """Create pd.DatetimeIndex with lower time resolution. + + The default resolution of pd.date_time is [ns], which puts boundaries on allowed start- and end-dates due to limited storage capacity. Choosing a resolution of [s] instead opens up that range considerably. + + "For nanosecond resolution, the time span that can be represented using a 64-bit integer is limited to approximately 584 years." + - https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timestamp-limitations + + You can check the available ranges with `pd.Timestamp.min` and `pd.Timestamp.max`. + """ + period_parts = re.search(r'^([0-9]+) *(.*)$', freq).groups() + if len(period_parts) != 2: + raise ValueError("Can't analyze period \"{period}\".") + number_str, unit = period_parts + start_date_np = np.datetime64(start, "s") + #print(f"start_date_np: {start_date_np}") + #print(to_numpy[unit]) + freq_np = np.timedelta64(int(number_str), to_numpy[unit]) + #print(f"freq_np: {freq_np}") + freq_pd = pd.Timedelta(freq_np) + #print(f"freq_pd: {freq_pd}") + + date_array = np.arange(start_date_np, start_date_np + periods * freq_pd, freq_pd) + date_array_with_frequency = pd.DatetimeIndex(date_array, freq=freq_pd, dtype="datetime64[s]") + + return date_array_with_frequency + def make_records( json_doc: dict | int | float | str, @@ -101,12 +135,12 @@ def make_records( make_records(val, {**idx_lvls, index_name: key}, res) case {"data": [float() | int(), *_] as data, "type": "time_series", **_r}: index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - ignore_year = json_doc.get(index, {}).get("ignore_year", None) - repeat = json_doc.get(index, {}).get("repeat", None) - if ignore_year == False: - raise ValueError('Can\'t handle `ignore_year == False`. Please re-format your dataset.') - if repeat == False: - raise ValueError('Can\'t handle `repeat == False`. Please re-format your dataset.') + ignore_year = json_doc.get("index", {}).get("ignore_year", None) + repeat = json_doc.get("index", {}).get("repeat", None) + if ignore_year == True: + raise ValueError('Can\'t handle `ignore_year == True`. Please re-format your dataset.') + if repeat == True: + raise ValueError('Can\'t handle `repeat == True`. Please re-format your dataset.') match json_doc: case { "index": { @@ -118,12 +152,12 @@ def make_records( **_r, }: freq = filter_frequencies(freq) - index = pd.date_range(start=start, freq=freq, periods=len(data)) + index = low_res_datetime(start=start, freq=freq, periods=len(data)) case _: - if json_doc.get(index, {}) != {}: + if json_doc.get("index", {}) != {}: raise NotImplementedError('Can\'t handle a partially set `index` value. Please re-format your dataset.') - index = pd.date_range( - start="0001-01-01", periods=len(data), freq="1h" + index = low_res_datetime( + start="0001-02-02", freq="1h", periods=len(data) ) for time, val in zip(index, data): make_records(val, {**idx_lvls, index_name: time, "value": val}, res) From e8f42ce6e956c967a7d7d92a799a9b2ecab02176 Mon Sep 17 00:00:00 2001 From: Ole Mussmann Date: Wed, 29 Jan 2025 18:42:25 +0100 Subject: [PATCH 02/43] add tests --- .../tests/integration/test_integration.py | 29 +++++++++++++++++++ arrow_expts/tests/unit/test_dbmap.py | 23 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 arrow_expts/tests/integration/test_integration.py create mode 100644 arrow_expts/tests/unit/test_dbmap.py diff --git a/arrow_expts/tests/integration/test_integration.py b/arrow_expts/tests/integration/test_integration.py new file mode 100644 index 0000000..6cca61c --- /dev/null +++ b/arrow_expts/tests/integration/test_integration.py @@ -0,0 +1,29 @@ +import json +import subprocess + +def test_array_numbers(): + input = """ + { + "type": "array", + "data": [2.3, 23.0, 5.0] + } + """.strip() + + output = """ + [ + { + "name": "i", + "values": [ + 2.3, + 23.0, + 5.0 + ], + "value_type": "number", + "type": "array" + } + ] + """.replace(" ","").replace("\n", "") + command = f"echo '{input}' | python -m schema.reencode" + result = subprocess.check_output(command, shell=True, text=True).strip() + + assert result == output diff --git a/arrow_expts/tests/unit/test_dbmap.py b/arrow_expts/tests/unit/test_dbmap.py new file mode 100644 index 0000000..5a75bf9 --- /dev/null +++ b/arrow_expts/tests/unit/test_dbmap.py @@ -0,0 +1,23 @@ +import numpy as np +import pandas as pd + +from ...spine.dbmap import \ + filter_frequencies, \ + low_res_datetime + +def test_filter_frequencies(): + assert filter_frequencies("5 years") == "5 Y" + assert filter_frequencies("5 year") == "5 Y" + assert filter_frequencies("3s") == "3s" + assert filter_frequencies("60") == "60m" + +def test_low_res_datetime(): + target = pd.date_range(start="2001-01-01", freq="7D", periods=5) + low_res = low_res_datetime(start="2001-01-01", freq="1W", periods=5) + + assert low_res.dtype == "datetime64[s]" + assert target.freq == low_res.freq + for date_target, date_low_res in zip(target, low_res): + assert date_target == date_low_res + + assert low_res_datetime(start="0001-01-01", freq="1s", periods=1)[0] == pd.Timestamp('0001-01-01') From 8808f2e46afa165b551585633a83f4a6a0dadc35 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 1 Apr 2025 15:21:12 +0200 Subject: [PATCH 03/43] Move tests out of package tree --- {arrow_expts/tests => tests}/integration/test_integration.py | 0 {arrow_expts/schema => tests}/json/array.durations.json | 0 {arrow_expts/schema => tests}/json/array.numbers.json | 0 {arrow_expts/schema => tests}/json/array.strings.json | 0 {arrow_expts/schema => tests}/json/date-time.json | 0 {arrow_expts/schema => tests}/json/duration.compact.json | 0 {arrow_expts/schema => tests}/json/duration.integer.json | 0 {arrow_expts/schema => tests}/json/duration.verbose.json | 0 {arrow_expts/schema => tests}/json/map.dictionary.json | 0 .../schema => tests}/json/map.stochastic-time-series-antti.json | 0 .../schema => tests}/json/map.stochastic-time-series.json | 0 {arrow_expts/schema => tests}/json/map.two-column-array.json | 0 {arrow_expts/schema => tests}/json/time-pattern.json | 0 {arrow_expts/schema => tests}/json/time-series.dict.json | 0 .../json/time-series.one-column-array-custom-indices.json | 0 .../schema => tests}/json/time-series.one-column-array.json | 0 .../json/time-series.two-column-array-named-indices.json | 0 .../schema => tests}/json/time-series.two-column-array.json | 0 {arrow_expts/tests => tests}/unit/test_dbmap.py | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename {arrow_expts/tests => tests}/integration/test_integration.py (100%) rename {arrow_expts/schema => tests}/json/array.durations.json (100%) rename {arrow_expts/schema => tests}/json/array.numbers.json (100%) rename {arrow_expts/schema => tests}/json/array.strings.json (100%) rename {arrow_expts/schema => tests}/json/date-time.json (100%) rename {arrow_expts/schema => tests}/json/duration.compact.json (100%) rename {arrow_expts/schema => tests}/json/duration.integer.json (100%) rename {arrow_expts/schema => tests}/json/duration.verbose.json (100%) rename {arrow_expts/schema => tests}/json/map.dictionary.json (100%) rename {arrow_expts/schema => tests}/json/map.stochastic-time-series-antti.json (100%) rename {arrow_expts/schema => tests}/json/map.stochastic-time-series.json (100%) rename {arrow_expts/schema => tests}/json/map.two-column-array.json (100%) rename {arrow_expts/schema => tests}/json/time-pattern.json (100%) rename {arrow_expts/schema => tests}/json/time-series.dict.json (100%) rename {arrow_expts/schema => tests}/json/time-series.one-column-array-custom-indices.json (100%) rename {arrow_expts/schema => tests}/json/time-series.one-column-array.json (100%) rename {arrow_expts/schema => tests}/json/time-series.two-column-array-named-indices.json (100%) rename {arrow_expts/schema => tests}/json/time-series.two-column-array.json (100%) rename {arrow_expts/tests => tests}/unit/test_dbmap.py (100%) diff --git a/arrow_expts/tests/integration/test_integration.py b/tests/integration/test_integration.py similarity index 100% rename from arrow_expts/tests/integration/test_integration.py rename to tests/integration/test_integration.py diff --git a/arrow_expts/schema/json/array.durations.json b/tests/json/array.durations.json similarity index 100% rename from arrow_expts/schema/json/array.durations.json rename to tests/json/array.durations.json diff --git a/arrow_expts/schema/json/array.numbers.json b/tests/json/array.numbers.json similarity index 100% rename from arrow_expts/schema/json/array.numbers.json rename to tests/json/array.numbers.json diff --git a/arrow_expts/schema/json/array.strings.json b/tests/json/array.strings.json similarity index 100% rename from arrow_expts/schema/json/array.strings.json rename to tests/json/array.strings.json diff --git a/arrow_expts/schema/json/date-time.json b/tests/json/date-time.json similarity index 100% rename from arrow_expts/schema/json/date-time.json rename to tests/json/date-time.json diff --git a/arrow_expts/schema/json/duration.compact.json b/tests/json/duration.compact.json similarity index 100% rename from arrow_expts/schema/json/duration.compact.json rename to tests/json/duration.compact.json diff --git a/arrow_expts/schema/json/duration.integer.json b/tests/json/duration.integer.json similarity index 100% rename from arrow_expts/schema/json/duration.integer.json rename to tests/json/duration.integer.json diff --git a/arrow_expts/schema/json/duration.verbose.json b/tests/json/duration.verbose.json similarity index 100% rename from arrow_expts/schema/json/duration.verbose.json rename to tests/json/duration.verbose.json diff --git a/arrow_expts/schema/json/map.dictionary.json b/tests/json/map.dictionary.json similarity index 100% rename from arrow_expts/schema/json/map.dictionary.json rename to tests/json/map.dictionary.json diff --git a/arrow_expts/schema/json/map.stochastic-time-series-antti.json b/tests/json/map.stochastic-time-series-antti.json similarity index 100% rename from arrow_expts/schema/json/map.stochastic-time-series-antti.json rename to tests/json/map.stochastic-time-series-antti.json diff --git a/arrow_expts/schema/json/map.stochastic-time-series.json b/tests/json/map.stochastic-time-series.json similarity index 100% rename from arrow_expts/schema/json/map.stochastic-time-series.json rename to tests/json/map.stochastic-time-series.json diff --git a/arrow_expts/schema/json/map.two-column-array.json b/tests/json/map.two-column-array.json similarity index 100% rename from arrow_expts/schema/json/map.two-column-array.json rename to tests/json/map.two-column-array.json diff --git a/arrow_expts/schema/json/time-pattern.json b/tests/json/time-pattern.json similarity index 100% rename from arrow_expts/schema/json/time-pattern.json rename to tests/json/time-pattern.json diff --git a/arrow_expts/schema/json/time-series.dict.json b/tests/json/time-series.dict.json similarity index 100% rename from arrow_expts/schema/json/time-series.dict.json rename to tests/json/time-series.dict.json diff --git a/arrow_expts/schema/json/time-series.one-column-array-custom-indices.json b/tests/json/time-series.one-column-array-custom-indices.json similarity index 100% rename from arrow_expts/schema/json/time-series.one-column-array-custom-indices.json rename to tests/json/time-series.one-column-array-custom-indices.json diff --git a/arrow_expts/schema/json/time-series.one-column-array.json b/tests/json/time-series.one-column-array.json similarity index 100% rename from arrow_expts/schema/json/time-series.one-column-array.json rename to tests/json/time-series.one-column-array.json diff --git a/arrow_expts/schema/json/time-series.two-column-array-named-indices.json b/tests/json/time-series.two-column-array-named-indices.json similarity index 100% rename from arrow_expts/schema/json/time-series.two-column-array-named-indices.json rename to tests/json/time-series.two-column-array-named-indices.json diff --git a/arrow_expts/schema/json/time-series.two-column-array.json b/tests/json/time-series.two-column-array.json similarity index 100% rename from arrow_expts/schema/json/time-series.two-column-array.json rename to tests/json/time-series.two-column-array.json diff --git a/arrow_expts/tests/unit/test_dbmap.py b/tests/unit/test_dbmap.py similarity index 100% rename from arrow_expts/tests/unit/test_dbmap.py rename to tests/unit/test_dbmap.py From 8daacea78fe4405bd1141d362e3ae10cbac01ebc Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 1 Apr 2025 15:21:39 +0200 Subject: [PATCH 04/43] Fix tests --- tests/integration/test_integration.py | 41 +++++++++++---------------- tests/unit/test_dbmap.py | 9 +++--- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 6cca61c..a485f5e 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -1,29 +1,20 @@ import json -import subprocess -def test_array_numbers(): - input = """ - { - "type": "array", - "data": [2.3, 23.0, 5.0] - } - """.strip() +import pytest - output = """ - [ - { - "name": "i", - "values": [ - 2.3, - 23.0, - 5.0 - ], - "value_type": "number", - "type": "array" - } - ] - """.replace(" ","").replace("\n", "") - command = f"echo '{input}' | python -m schema.reencode" - result = subprocess.check_output(command, shell=True, text=True).strip() +from arrow_expts.schema.models import Array +from arrow_expts.schema.reencode import series_to_col, to_df, to_tables - assert result == output +from ..conftest import JSONDIR + + +@pytest.mark.parametrize("fname,arr", [("array.numbers.json", [2.3, 23.0, 5.0])]) +def test_json_to_tbl(fname, arr): + input_json = JSONDIR / fname + data = json.loads(input_json.read_text()) + + exp = Array(name="i", values=arr) + df = to_df(data) + + assert series_to_col(df[exp.name]) == exp + assert to_tables(df) == [exp] diff --git a/tests/unit/test_dbmap.py b/tests/unit/test_dbmap.py index 5a75bf9..6e94ba3 100644 --- a/tests/unit/test_dbmap.py +++ b/tests/unit/test_dbmap.py @@ -1,9 +1,8 @@ import numpy as np import pandas as pd -from ...spine.dbmap import \ - filter_frequencies, \ - low_res_datetime +from arrow_expts.spine.dbmap import filter_frequencies, low_res_datetime + def test_filter_frequencies(): assert filter_frequencies("5 years") == "5 Y" @@ -11,6 +10,7 @@ def test_filter_frequencies(): assert filter_frequencies("3s") == "3s" assert filter_frequencies("60") == "60m" + def test_low_res_datetime(): target = pd.date_range(start="2001-01-01", freq="7D", periods=5) low_res = low_res_datetime(start="2001-01-01", freq="1W", periods=5) @@ -20,4 +20,5 @@ def test_low_res_datetime(): for date_target, date_low_res in zip(target, low_res): assert date_target == date_low_res - assert low_res_datetime(start="0001-01-01", freq="1s", periods=1)[0] == pd.Timestamp('0001-01-01') + res = low_res_datetime(start="0001-01-01", freq="1s", periods=1) + assert res[0] == pd.Timestamp("0001-01-01") From b5854f111e4df880a950d900cd511f26381bd23f Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 1 Apr 2025 15:22:20 +0200 Subject: [PATCH 05/43] reencode.py: fix to match model rename, and some refactor --- arrow_expts/schema/reencode.py | 122 ++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index 5251e0e..435c5fa 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -15,6 +15,7 @@ from datetime import datetime, timedelta import json from pathlib import Path +from typing import cast, overload import pandas as pd import pyarrow as pa @@ -26,19 +27,22 @@ from .models import ( Array, ArrayIndex, - DEArray, - DEIndex, - REArray, - REIndex, - RLIndex, + DictEncodedArray, + DictEncodedIndex, + RunEndArray, + RunEndIndex, + RunLengthArray, + RunLengthIndex, Table, ) def to_df(json_doc: dict): data = make_records(json_doc, {}, [], idx_name="metric") - tbl = pa.Table.from_pylist(data) - df = tbl.to_pandas(types_mapper=pd.ArrowDtype) + # NOTE: don't use pyarrow, difficult to support mixed types + # tbl = pa.Table.from_pylist(data) + # df = tbl.to_pandas(types_mapper=pd.ArrowDtype) + df = pd.DataFrame.from_records(data) return df @@ -46,70 +50,100 @@ class _sentinel: pass -def rl_encode(arr: ArrayIndex) -> RLIndex: - last = _sentinel() - values = [] - run_len = [] +SENTINEL = _sentinel() + + +@overload +def rl_encode(arr: Array) -> RunLengthArray: ... + + +@overload +def rl_encode(arr: ArrayIndex) -> RunLengthIndex: ... + + +def rl_encode(arr: ArrayIndex | Array) -> RunLengthIndex | RunLengthArray: + last = SENTINEL + values, run_len = [], [] for val in arr.values: if val != last: values.append(val) run_len.append(1) + last = val else: run_len[-1] += 1 - return RLIndex(name=arr.name, values=values, run_len=run_len) + return RunLengthIndex(name=arr.name, values=values, run_len=run_len) -def re_encode(arr: ArrayIndex) -> REIndex: - last = arr.values[0] # _sentinel() - values = [last] - run_end = [] - for idx, val in enumerate(arr.values[1:], start=1): - if val != last: - last = val +@overload +def re_encode(arr: Array) -> RunEndArray: ... + + +@overload +def re_encode(arr: ArrayIndex) -> RunEndIndex: ... + + +def re_encode(arr: ArrayIndex | Array) -> RunEndIndex | RunEndArray: + last = SENTINEL + values, run_end = [], [] + for idx, val in enumerate(arr.values, start=1): + if last != val: values.append(val) run_end.append(idx) - run_end.append(len(arr.values)) - return REIndex(name=arr.name, values=values, run_end=run_end) + else: + run_end[-1] = idx + last = val + return RunEndIndex(name=arr.name, values=values, run_end=run_end) + + +@overload +def de_encode(arr: Array) -> DictEncodedArray: ... + +@overload +def de_encode(arr: ArrayIndex) -> DictEncodedIndex: ... -def de_encode(arr: ArrayIndex) -> DEIndex: + +def de_encode(arr: ArrayIndex | Array) -> DictEncodedIndex | DictEncodedArray: # not using list(set(...)) to preserve order values = list(dict.fromkeys(arr.values)) indices = list(map(values.index, arr.values)) - return DEIndex(name=arr.name, values=values, indices=indices) + return DictEncodedIndex(name=arr.name, values=values, indices=indices) -def series_to_col(col: pd.Series) -> ArrayIndex | DEIndex | Array | DEArray: +def series_to_col( + col: pd.Series, +) -> ArrayIndex | DictEncodedIndex | Array | DictEncodedArray: match col.name, col.dtype.type: - case "value", t if issubclass(t, bool | str) or t is object: - print(f"idx_type: {t}, value: {col.iloc[:3]}") + case "value", t if issubclass(t, str) or t is object: + print(f"type: {t}, value: {col.iloc[:3]}") col = col.astype("category") - arr = DEArray( + return DictEncodedArray( name=col.name, values=col.cat.categories, indices=col.cat.codes, ) - case _, t if issubclass(t, int | float): - arr = Array(name=col.name, values=col.values) + case "value", t if issubclass(t, int): + return Array(name=col.name, values=col.values) + case _, t if issubclass(t, (bool, float, bytes)): + return Array(name=col.name, values=col.values) case _, t if issubclass(t, str) or t is object: - print(f"type: {t}, value: {col.iloc[:3]}") + print(f"idx_type: {t}, value: {col.iloc[:3]}") col = col.astype("category") - arr = DEIndex( + return DictEncodedIndex( name=col.name, values=col.cat.categories, indices=col.cat.codes, ) - case _, t if issubclass(t, int | str | datetime | timedelta) or t is object: - arr = ArrayIndex(name=col.name, values=col.values) - case _, _: - raise NotImplementedError(f"unknown type {t}") - return arr + case _, t if issubclass(t, (int, datetime, timedelta)) or t is object: + return ArrayIndex(name=col.name, values=col.values) + case n, t: + raise NotImplementedError(f"{n}: unknown type {t}") def to_tables(df: pd.DataFrame) -> Table: if df.empty: return [] - return [series_to_col(df[colname]) for colname in df.columns] + return [series_to_col(col) for _, col in df.items()] if __name__ == "__main__": @@ -117,20 +151,12 @@ def to_tables(df: pd.DataFrame) -> Table: parser = ArgumentParser(__doc__) parser.add_argument("old_json") - #parser.add_argument("new_json") + parser.add_argument("new_json") opts = parser.parse_args() - new_json = opts.old_json.replace("/0_orig/", "/1_reencoded/") # FIXME brittle df = to_df(json.loads(Path(opts.old_json).read_text())) - print("== DF ==") - #print(df) tbls = to_tables(df) - print("== TBLS ==") - print(tbls) - model = RootModel[Table](tbls).model_dump_json(indent=2) - print("== MODEL ==") - print(model) # NOTE: using pydantic is optional here, we can also write our own # JSON serialisation if we want, probably all we need is `asdict(...)`. - #Path(opts.new_json).write_text(RootModel[Table](tbls).model_dump_json()) - Path(new_json).write_text(model) + json_blob = RootModel[Table](tbls).model_dump_json(indent=2) + Path(opts.new_json).write_text(json_blob) From aad2161461d6722609dde13838408354cfa1012a Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 1 Apr 2025 15:22:53 +0200 Subject: [PATCH 06/43] models.py: add bytes to support `Any` (flexible) data types --- arrow_expts/schema/models.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index ba28c92..9185b02 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -14,7 +14,7 @@ # from dataclasses import dataclass # from dataclasses import field from datetime import datetime, timedelta -from typing import Annotated, Literal, Type, TypeAlias +from typing import Annotated, Literal, TypeAlias import pandas as pd from pydantic import RootModel @@ -27,6 +27,7 @@ Integers: TypeAlias = list[int] Strings: TypeAlias = list[str] Booleans: TypeAlias = list[bool] +BytesList: TypeAlias = list[bytes] Datetimes: TypeAlias = list[datetime] Timedeltas: TypeAlias = list[timedelta] @@ -46,7 +47,14 @@ IndexTypes: TypeAlias = Integers | Strings | Datetimes | Timedeltas | TimePatterns ValueTypes: TypeAlias = ( - Integers | Strings | Floats | Booleans | Datetimes | Timedeltas | TimePatterns + Integers + | Strings + | Floats + | Booleans + | Datetimes + | Timedeltas + | TimePatterns + | BytesList ) NullableValueTypes: TypeAlias = ( NullableIntegers @@ -76,6 +84,7 @@ timedelta: "duration", pd.Timedelta: "duration", TimePattern: "time-pattern", + bytes: "string", } @@ -182,11 +191,27 @@ class Array(_TypeInferMixin): type: Literal["array"] = "array" +@dataclass(frozen=True) +class BytesArray(_TypeInferMixin): + """Array of bytes to store mixed types""" + + name: str + values: BytesList + value_type: ValueTypeNames = field(init=False) + type: Literal["bytes_array"] = "bytes_array" + + # NOTE: To add run-length encoding to the schema, add it to the # following type union following which, we need to implement a # converter to an Arrow array type Table: TypeAlias = list[ - RunEndIndex | DictEncodedIndex | ArrayIndex | RunEndArray | DictEncodedArray | Array + RunEndIndex + | DictEncodedIndex + | ArrayIndex + | RunEndArray + | DictEncodedArray + | Array + | BytesArray ] From d5db0e67ead63391a2b7f8bcb630462a671b6406 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 1 Apr 2025 15:24:00 +0200 Subject: [PATCH 07/43] dbmap.py: major refactor --- arrow_expts/spine/dbmap.py | 313 ++++++++++++++++++------------------- tests/unit/test_dbmap.py | 10 +- 2 files changed, 155 insertions(+), 168 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 1002055..bd2eeeb 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -1,9 +1,9 @@ from argparse import ArgumentParser from datetime import datetime -from enum import Enum, auto import json import re -from typing import cast +from typing import cast, Any, Callable, Iterable, TypeAlias +from warnings import warn import weakref import pandas as pd @@ -18,34 +18,33 @@ def json_loads_ts(json_str: str | bytes): SEQ_PAT = re.compile(r"(t|p)([0-9]+)") +FREQ_PAT = re.compile("^[0-9]+$") -def filter_frequencies(freq: str) -> str: + +def normalise_freq(freq: int | str): + if isinstance(freq, int): + return str(freq) + "m" + if FREQ_PAT.match(freq): + # If frequency is an integer, the implied unit is "minutes" + return freq + "m" # not very robust yet - filtered_freq = freq \ - .replace("years", "Y") \ - .replace("year", "Y") \ - .replace("months", "M") \ - .replace("month", "M") \ - .replace("weeks", "W") \ - .replace("week", "W") \ - .replace("days", "D") \ - .replace("day", "D") \ - .replace("hours", "h") \ - .replace("hour", "h") \ - .replace("minutes", "min") \ - .replace("minute", "min") \ - .replace("seconds", "s") \ + return ( + freq.replace("years", "Y") + .replace("year", "Y") + .replace("months", "M") + .replace("month", "M") + .replace("weeks", "W") + .replace("week", "W") + .replace("days", "D") + .replace("day", "D") + .replace("hours", "h") + .replace("hour", "h") + .replace("minutes", "min") + .replace("minute", "min") + .replace("seconds", "s") .replace("second", "s") - if re.compile("^[0-9]+$").match(filtered_freq): - # If frequency is an integer, the implied unit is "minutes" - return filtered_freq + "m" - else: - return filtered_freq + ) -class IndexType(Enum): - Timestamp = auto() - Sequence = auto() - Generic = auto() to_numpy = { "Y": "Y", @@ -57,34 +56,78 @@ class IndexType(Enum): "s": "s", } + def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: """Create pd.DatetimeIndex with lower time resolution. - The default resolution of pd.date_time is [ns], which puts boundaries on allowed start- and end-dates due to limited storage capacity. Choosing a resolution of [s] instead opens up that range considerably. + The default resolution of pd.date_time is [ns], which puts + boundaries on allowed start- and end-dates due to limited storage + capacity. Choosing a resolution of [s] instead opens up that range + considerably. + + "For nanosecond resolution, the time span that can be represented + using a 64-bit integer is limited to approximately 584 years." - + https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timestamp-limitations - "For nanosecond resolution, the time span that can be represented using a 64-bit integer is limited to approximately 584 years." - - https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timestamp-limitations + You can check the available ranges with `pd.Timestamp.min` and + `pd.Timestamp.max`. - You can check the available ranges with `pd.Timestamp.min` and `pd.Timestamp.max`. """ - period_parts = re.search(r'^([0-9]+) *(.*)$', freq).groups() + if re_match := re.search(r"^([0-9]+) *(.*)$", freq): + period_parts = re_match.groups() + else: + raise ValueError(f"invalid frequency: {freq!r}") + if len(period_parts) != 2: - raise ValueError("Can't analyze period \"{period}\".") + raise ValueError(f"invalid frequency: {freq!r}") + number_str, unit = period_parts start_date_np = np.datetime64(start, "s") - #print(f"start_date_np: {start_date_np}") - #print(to_numpy[unit]) + # print(f"start_date_np: {start_date_np}") + # print(to_numpy[unit]) freq_np = np.timedelta64(int(number_str), to_numpy[unit]) - #print(f"freq_np: {freq_np}") + # print(f"freq_np: {freq_np}") freq_pd = pd.Timedelta(freq_np) - #print(f"freq_pd: {freq_pd}") + # print(f"freq_pd: {freq_pd}") - date_array = np.arange(start_date_np, start_date_np + periods * freq_pd, freq_pd) - date_array_with_frequency = pd.DatetimeIndex(date_array, freq=freq_pd, dtype="datetime64[s]") + date_array = np.arange(start_date_np, start_date_np + periods * freq_np, freq_np) + date_array_with_frequency = pd.DatetimeIndex( + date_array, freq=freq_pd, dtype="datetime64[s]" + ) return date_array_with_frequency +def _atoi(name: str, val: str) -> dict[str, int | str]: + """Convert string to number if it matches `t0001` or `p2001`.""" + if m := SEQ_PAT.match(val): + name = "period" if "p" == m.group(1) else "time" + return {name: int(m.group(2))} + else: + return {name: val} + + +_FmtIdx: TypeAlias = ( + Callable[[str, str], dict[str, Any]] | Callable[[str, dict], dict[str, Any]] +) + + +def _formatter(index_type: str | dict) -> _FmtIdx: + match index_type: + case "date_time" | "datetime": + return lambda name, key: {name: datetime.fromisoformat(key)} + case "duration": + return lambda name, key: {name: normalise_freq(key)} + case "str": + # custom handling when data matches `SEQ_PAT` + return _atoi + case "float" | "time_pattern" | "timepattern" | "noop": + return lambda name, key: {name: key} + case _: # fallback to noop w/ a warning + warn(f"{index_type}: unknown type, fallback to noop formatter") + return lambda name, key: {name: key} + + def make_records( json_doc: dict | int | float | str, idx_lvls: dict, @@ -98,141 +141,85 @@ def make_records( """ + def _from_pairs(data: Iterable[Iterable], fmt: _FmtIdx): + assert isinstance(json_doc, dict) + index_name = json_doc.get("index_name", idx_name) + for key, val in data: + make_records(val, {**idx_lvls, **fmt(index_name, key)}, res) + + def _deprecated(var: str, val: Any): + assert isinstance(json_doc, dict) + index_name = json_doc.get("index_name") + msg = f"{index_name}: {var}={val} is deprecated, handle in model, defaulting to time index from 0001-01-01." + warn(msg, DeprecationWarning) + + def _time_index(idx: dict, length: int): + start = idx.get("start", "0001-01-01T00:00:00") + resolution = idx.get("resolution", "1h") + freq = normalise_freq(resolution) + return low_res_datetime(start=start, freq=freq, periods=length) + + def _append_arr(arr: Iterable, fmt: _FmtIdx): + assert isinstance(json_doc, dict) + index_name = json_doc.get("index_name", "i") + for value in arr: + res.append({**idx_lvls, **fmt(index_name, value)}) + + def _append_val(value, fmt: _FmtIdx): + res.append({**idx_lvls, **fmt("value", value)}) + match json_doc: # maps - case {"data": list() as data, "type": "map", **_r}: - index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - index_type = json_doc.get("index_type") - for key, val in data: - if index_type == "date_time": - key = datetime.fromisoformat(key) - if index_type == "duration": - key = pd.Timedelta(key) - make_records(val, {**idx_lvls, index_name: key}, res) - case {"data": dict() as data, "type": "map", **_r}: - index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - index_type = json_doc.get("index_type") - for key, val in data.items(): - if index_type == "date_time": - key = datetime.fromisoformat(key) - if index_type == "duration": - key = pd.Timedelta(key) - make_records(val, {**idx_lvls, index_name: key}, res) + case {"data": dict() as data, "type": "map", "index_type": index_type}: + _from_pairs(data.items(), _formatter(index_type)) + case {"data": dict() as data, "index_type": index_type}: + _from_pairs(data.items(), _formatter(index_type)) + case {"data": [[_, _], *_] as data, "type": "map", "index_type": index_type}: + _from_pairs(data, _formatter(index_type)) + case {"data": [[_, _], *_] as data, "index_type": index_type}: + _from_pairs(data, _formatter(index_type)) # time series - case {"data": dict() as data, "type": "time_series", **_r}: - index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - for key, val in data.items(): - key = datetime.fromisoformat(key) - make_records(val, {**idx_lvls, index_name: key}, res) + case {"data": dict() as data, "type": "time_series"}: + _from_pairs(data.items(), _formatter("date_time")) + case {"data": [[str(), float() | int()], *_] as data, "type": "time_series"}: + _from_pairs(data, _formatter("date_time")) case { - "data": [[str(), float() | int()], *_] as data, + "data": [float() | int(), *_] as data, "type": "time_series", - **_r, + "index": dict() as idx, + }: + match idx: + case {"ignore_year": ignore_year}: + _deprecated("ignore_year", ignore_year) + case {"repeat": repeat}: + _deprecated("repeat", repeat) + + index = _time_index(idx, len(data)) + _from_pairs(zip(index, data), _formatter("noop")) + case {"type": "time_series", "data": [float() | int(), *_] as data}: + _append_arr(data, _formatter("noop")) + # arrays + case { + "type": "array", + "value_type": value_type, + "data": [str() | float() | int(), *_] as data, }: - index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - for key, val in data: - key = datetime.fromisoformat(key) - make_records(val, {**idx_lvls, index_name: key}, res) - case {"data": [float() | int(), *_] as data, "type": "time_series", **_r}: - index_name = json_doc.get("index_name", "time") # use "time" if "index_name" does not exist - ignore_year = json_doc.get("index", {}).get("ignore_year", None) - repeat = json_doc.get("index", {}).get("repeat", None) - if ignore_year == True: - raise ValueError('Can\'t handle `ignore_year == True`. Please re-format your dataset.') - if repeat == True: - raise ValueError('Can\'t handle `repeat == True`. Please re-format your dataset.') - match json_doc: - case { - "index": { - "start": start, - "resolution": freq, - "ignore_year": bool(), - "repeat": bool(), - }, - **_r, - }: - freq = filter_frequencies(freq) - index = low_res_datetime(start=start, freq=freq, periods=len(data)) - case _: - if json_doc.get("index", {}) != {}: - raise NotImplementedError('Can\'t handle a partially set `index` value. Please re-format your dataset.') - index = low_res_datetime( - start="0001-02-02", freq="1h", periods=len(data) - ) - for time, val in zip(index, data): - make_records(val, {**idx_lvls, index_name: time, "value": val}, res) + _append_arr(data, _formatter(value_type)) + case {"type": "array", "data": [float() | int(), *_] as data}: + if (value_type := json_doc.get("value_type", "float")) != "float": + raise ValueError(f"{value_type=}: unknown type in array: {data[:2]}") + _append_arr(data, _formatter(value_type)) + # date_time | duration | time_pattern case { - "data": [[str(), dict() | float() | int()], *_] as data, - "type": "time_series", - **_r, + "type": "date_time" | "duration" | "time_pattern" as data_t, + "data": str() | int() as data, }: - if m := SEQ_PAT.match(data[0][0]): - idx_type = IndexType.Sequence - index_name = json_doc.get( - "index_name", "period" if "p" == m.group(1) else "seq" - ) - else: - idx_type = IndexType.Generic - index_name = json_doc.get("index_name", idx_name) # use idx_name if "index_name" does not exist - for key, val in data: - if idx_type == IndexType.Sequence: - m = SEQ_PAT.match(key) - assert m is not None - key = int(m.group(2)) - make_records(val, {**idx_lvls, index_name: key}, res) - - # arrays - case {"type": "array", "data": [str() | float() | int(), *_] as data, **_r}: - value_type = json_doc.get("value_type", "float") # use "float" if "value_type" does not exist - index_name = json_doc.get("index_name", "i") # use "i" if "index_name" does not exist - - if value_type == "duration": - try: - data = [pd.Timedelta(filter_frequencies(value)) for value in data] - except ValueError as err: - if "invalid unit abbreviation" in repr(err): - raise ValueError('"year" and "month" are ambiguous time units. Please convert them to days.') - else: - raise err - elif value_type == "date_time": - data = [datetime.fromisoformat(key) for key in data] - elif value_type == "float": - data = [pd.to_numeric(value) for value in data] - elif value_type == "str": - pass - else: - raise NotImplementedError(f"Can't match {value_type} arrays.") - for value in data: - res.append({index_name: value}) - - # date-time - case {"type": "date_time", "data": str() as data, **_r}: - idx_lvls["value"] = datetime.fromisoformat(data) - res.append(idx_lvls) - - # duration - case {"type": "duration", "data": int() | str() as data, **_r}: - if type(data) == int: - data = str(data) + "m" # integer time unit is "minutes" - try: - data = pd.Timedelta(filter_frequencies(data)) - except ValueError as err: - if "invalid unit abbreviation" in repr(err): - raise ValueError('"year" and "month" are ambiguous time units. Please convert them to days.') - else: - raise err - res.append({"value": data}) - - # time_pattern - case {"type": "time_pattern", **_r}: - raise NotImplementedError("Can't convert `time_pattern`. Please convert it to a `time_series`.") - + _append_val(data, _formatter(data_t)) # values - case int() | float() | str() | bool(): - idx_lvls["value"] = json_doc - res.append(idx_lvls) + case int() | float() | str() | bool() as data: + _append_val(data, _formatter("noop")) case _: - raise NotImplementedError("Can't match this JSON structure yet.") + raise ValueError(f"match not found: {json_doc}") return res diff --git a/tests/unit/test_dbmap.py b/tests/unit/test_dbmap.py index 6e94ba3..1cdf7a1 100644 --- a/tests/unit/test_dbmap.py +++ b/tests/unit/test_dbmap.py @@ -1,14 +1,14 @@ import numpy as np import pandas as pd -from arrow_expts.spine.dbmap import filter_frequencies, low_res_datetime +from arrow_expts.spine.dbmap import normalise_freq, low_res_datetime def test_filter_frequencies(): - assert filter_frequencies("5 years") == "5 Y" - assert filter_frequencies("5 year") == "5 Y" - assert filter_frequencies("3s") == "3s" - assert filter_frequencies("60") == "60m" + assert normalise_freq("5 years") == "5 Y" + assert normalise_freq("5 year") == "5 Y" + assert normalise_freq("3s") == "3s" + assert normalise_freq("60") == "60m" def test_low_res_datetime(): From edb587c5d64d053353ddd03cd68cb4b809224fac Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 4 Apr 2025 12:02:10 +0200 Subject: [PATCH 08/43] dbmap: fix _formatter type hints --- arrow_expts/spine/dbmap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index bd2eeeb..1f5dd36 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -107,12 +107,10 @@ def _atoi(name: str, val: str) -> dict[str, int | str]: return {name: val} -_FmtIdx: TypeAlias = ( - Callable[[str, str], dict[str, Any]] | Callable[[str, dict], dict[str, Any]] -) +_FmtIdx: TypeAlias = Callable[[str, str | Any], dict[str, Any]] -def _formatter(index_type: str | dict) -> _FmtIdx: +def _formatter(index_type: str) -> _FmtIdx: match index_type: case "date_time" | "datetime": return lambda name, key: {name: datetime.fromisoformat(key)} From 235418db6d73bdc691844f2ed84c5fa34b2482bf Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 4 Apr 2025 12:03:35 +0200 Subject: [PATCH 09/43] dbmap: simplify append for single values --- arrow_expts/spine/dbmap.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 1f5dd36..41c9453 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -163,9 +163,6 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): for value in arr: res.append({**idx_lvls, **fmt(index_name, value)}) - def _append_val(value, fmt: _FmtIdx): - res.append({**idx_lvls, **fmt("value", value)}) - match json_doc: # maps case {"data": dict() as data, "type": "map", "index_type": index_type}: @@ -212,10 +209,12 @@ def _append_val(value, fmt: _FmtIdx): "type": "date_time" | "duration" | "time_pattern" as data_t, "data": str() | int() as data, }: - _append_val(data, _formatter(data_t)) + _fmt = _formatter(data_t) + res.append({**idx_lvls, **_fmt("value", data)}) # values case int() | float() | str() | bool() as data: - _append_val(data, _formatter("noop")) + _fmt = _formatter("noop") + res.append({**idx_lvls, **_fmt("value", data)}) case _: raise ValueError(f"match not found: {json_doc}") return res From cc6853341b91e1b5651cf90157dbf78394fd4c85 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 4 Apr 2025 12:04:13 +0200 Subject: [PATCH 10/43] dbmap: more robust handling of "index_type" for "map"s --- arrow_expts/spine/dbmap.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 41c9453..dee6334 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -165,11 +165,19 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): match json_doc: # maps - case {"data": dict() as data, "type": "map", "index_type": index_type}: + case {"data": dict() as data, "type": "map"}: + # NOTE: is "index_type" mandatory? In case it's not, we + # check for it separately, and fallback in a way that + # raises a warning but doesn't crash; same for the + # 2-column array variant below. + index_type = json_doc.get("index_type", "undefined-index_type-in-map") _from_pairs(data.items(), _formatter(index_type)) case {"data": dict() as data, "index_type": index_type}: + # NOTE: relies on other types not having "index_type"; + # same for the 2-column array variant below. _from_pairs(data.items(), _formatter(index_type)) - case {"data": [[_, _], *_] as data, "type": "map", "index_type": index_type}: + case {"data": [[_, _], *_] as data, "type": "map"}: + index_type = json_doc.get("index_type", "undefined-index_type-in-map") _from_pairs(data, _formatter(index_type)) case {"data": [[_, _], *_] as data, "index_type": index_type}: _from_pairs(data, _formatter(index_type)) From e9c9540f0bd5e8cf71670eccb2782b4569c5f5fc Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 4 Apr 2025 12:04:59 +0200 Subject: [PATCH 11/43] reencode: clean up unused imports, undo index name default --- arrow_expts/schema/reencode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index 435c5fa..50d7b15 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -15,7 +15,7 @@ from datetime import datetime, timedelta import json from pathlib import Path -from typing import cast, overload +from typing import overload import pandas as pd import pyarrow as pa @@ -38,7 +38,7 @@ def to_df(json_doc: dict): - data = make_records(json_doc, {}, [], idx_name="metric") + data = make_records(json_doc, {}, []) # NOTE: don't use pyarrow, difficult to support mixed types # tbl = pa.Table.from_pylist(data) # df = tbl.to_pandas(types_mapper=pd.ArrowDtype) From 05c93cc1421a4bd5a8b66d035a5f9956e8c8631d Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 4 Apr 2025 13:05:17 +0200 Subject: [PATCH 12/43] dbmap: add docs, move tests --- arrow_expts/spine/dbmap.py | 63 ++++++++++++++++++++- tests/{unit => }/test_dbmap.py | 0 tests/{integration => }/test_integration.py | 2 +- 3 files changed, 61 insertions(+), 4 deletions(-) rename tests/{unit => }/test_dbmap.py (100%) rename tests/{integration => }/test_integration.py (93%) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index dee6334..7ea183b 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -17,11 +17,20 @@ def json_loads_ts(json_str: str | bytes): return pd.Series(json.loads(json_str)["data"]) +# Regex pattern to indentify numerical sequences encoded as string SEQ_PAT = re.compile(r"(t|p)([0-9]+)") +# Regex pattern to identify a number encoded as a string FREQ_PAT = re.compile("^[0-9]+$") def normalise_freq(freq: int | str): + """Normalise integer/string to frequency. + + The frequency value is as understood by `pandas.Timedelta`. Note + that ambiguous values such as month or year are still retained + with the intention to handle later in the pipeline. + + """ if isinstance(freq, int): return str(freq) + "m" if FREQ_PAT.match(freq): @@ -99,7 +108,12 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: def _atoi(name: str, val: str) -> dict[str, int | str]: - """Convert string to number if it matches `t0001` or `p2001`.""" + """Convert string to number if it matches `t0001` or `p2001`. + + If a match is found, also override the name to "time" or "period" + respectively. + + """ if m := SEQ_PAT.match(val): name = "period" if "p" == m.group(1) else "time" return {name: int(m.group(2))} @@ -111,6 +125,32 @@ def _atoi(name: str, val: str) -> dict[str, int | str]: def _formatter(index_type: str) -> _FmtIdx: + """Get a function that formats the values of a name value pair. + + The name is the column name. The function returned depends on the + `index_type`. An unknown `index_type` returns a noop formatter, + but it also issues a warning. A noop formatter can be requested + explicitly by passing the type "noop"; no warning is issued in + this case. + + Index types: + ============ + + - "date_time" :: converts value to `datetime` + + - "duration" :: converts string to `pandas.Timedelta` compatible + argument; note it still allows for ambiguous units like month or + year. + + - "str" :: convert the value to integer if it matches `t0001` or + `p2002`, and the name to "time" and "period" respectively; + without a match it is a noop. + + - "float" | "time_pattern" | "noop" :: noop + + - fallback :: noop with a warning + + """ match index_type: case "date_time" | "datetime": return lambda name, key: {name: datetime.fromisoformat(key)} @@ -133,12 +173,29 @@ def make_records( *, idx_name: str = "default", ) -> list[dict]: - """Parse time-series w/ a multi-index stored as a nested map + """Parse parameter value into a list of records + + Spine db stores parameter_value as JSON. After the JSON blob has + been decoded to a Python dict, this function can transform it into + a list of records (dict) like a table. These records can then be + consumed by Pandas to create a dataframe. + + The parsing logic works recursively by traversing depth first. + Each call incrementally accumulates a cell/level of a record in + the `idx_lvls` dictionary, once the traversal reaches a leaf node, + the final record is appended to the list `res`. The final result + is also returned by the function, allowing for composition. - Ask Suvayu for the example DB + If at any level, the index name is missing, a default can be + provided by setting a default `idx_name`. """ + # NOTE: The private functions below are closures, defined early in + # the function such that they have the original arguments to + # `make_records` available to them, but nothing more. They either + # help with some computation, raise a warning, or are helpers to + # append to the result. def _from_pairs(data: Iterable[Iterable], fmt: _FmtIdx): assert isinstance(json_doc, dict) index_name = json_doc.get("index_name", idx_name) diff --git a/tests/unit/test_dbmap.py b/tests/test_dbmap.py similarity index 100% rename from tests/unit/test_dbmap.py rename to tests/test_dbmap.py diff --git a/tests/integration/test_integration.py b/tests/test_integration.py similarity index 93% rename from tests/integration/test_integration.py rename to tests/test_integration.py index a485f5e..163aa91 100644 --- a/tests/integration/test_integration.py +++ b/tests/test_integration.py @@ -5,7 +5,7 @@ from arrow_expts.schema.models import Array from arrow_expts.schema.reencode import series_to_col, to_df, to_tables -from ..conftest import JSONDIR +from .conftest import JSONDIR @pytest.mark.parametrize("fname,arr", [("array.numbers.json", [2.3, 23.0, 5.0])]) From 20da49914ad0d3a69313e350fd27221d1193b581 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 8 Apr 2025 15:24:28 +0200 Subject: [PATCH 13/43] dbmap: fix how default index level names are handled --- arrow_expts/spine/dbmap.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 7ea183b..c529ab9 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -171,7 +171,7 @@ def make_records( idx_lvls: dict, res: list[dict], *, - idx_name: str = "default", + lvlname_base: str = "default", ) -> list[dict]: """Parse parameter value into a list of records @@ -186,10 +186,13 @@ def make_records( the final record is appended to the list `res`. The final result is also returned by the function, allowing for composition. - If at any level, the index name is missing, a default can be - provided by setting a default `idx_name`. + If at any level, the index level name is missing, a default base + name can be provided by setting a default `lvlname_base`. The + level name is derived by concatenating the base name with depth + level. """ + lvlname = lvlname_base + f"{len(idx_lvls)}" # NOTE: The private functions below are closures, defined early in # the function such that they have the original arguments to @@ -198,9 +201,10 @@ def make_records( # append to the result. def _from_pairs(data: Iterable[Iterable], fmt: _FmtIdx): assert isinstance(json_doc, dict) - index_name = json_doc.get("index_name", idx_name) + index_name = json_doc.get("index_name", lvlname) for key, val in data: - make_records(val, {**idx_lvls, **fmt(index_name, key)}, res) + _lvls = {**idx_lvls, **fmt(index_name, key)} + make_records(val, _lvls, res, lvlname_base=lvlname_base) def _deprecated(var: str, val: Any): assert isinstance(json_doc, dict) From 1c8e97c8e8629c38b2d735944ac229e1f6f71063 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Tue, 8 Apr 2025 15:25:01 +0200 Subject: [PATCH 14/43] dbmap: remove redundant value_type check from a case --- arrow_expts/spine/dbmap.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index c529ab9..bbbcbae 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -270,9 +270,7 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): }: _append_arr(data, _formatter(value_type)) case {"type": "array", "data": [float() | int(), *_] as data}: - if (value_type := json_doc.get("value_type", "float")) != "float": - raise ValueError(f"{value_type=}: unknown type in array: {data[:2]}") - _append_arr(data, _formatter(value_type)) + _append_arr(data, _formatter("float")) # date_time | duration | time_pattern case { "type": "date_time" | "duration" | "time_pattern" as data_t, From c18f3ee1934673476ca95fe0e2885d4da9990728 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:07:42 +0200 Subject: [PATCH 15/43] dbmap: fix index_name in deprecation warning --- arrow_expts/spine/dbmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index bbbcbae..d904cca 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -208,7 +208,7 @@ def _from_pairs(data: Iterable[Iterable], fmt: _FmtIdx): def _deprecated(var: str, val: Any): assert isinstance(json_doc, dict) - index_name = json_doc.get("index_name") + index_name = json_doc.get("index_name", lvlname) msg = f"{index_name}: {var}={val} is deprecated, handle in model, defaulting to time index from 0001-01-01." warn(msg, DeprecationWarning) From 078428b876022e85825a988603d988465cab2e30 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:31:24 +0200 Subject: [PATCH 16/43] dbmap: simplify underlying formatter function signature for testing --- arrow_expts/spine/dbmap.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index d904cca..5a5c760 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -107,7 +107,7 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: return date_array_with_frequency -def _atoi(name: str, val: str) -> dict[str, int | str]: +def _atoi(val: str) -> int | str: """Convert string to number if it matches `t0001` or `p2001`. If a match is found, also override the name to "time" or "period" @@ -116,9 +116,9 @@ def _atoi(name: str, val: str) -> dict[str, int | str]: """ if m := SEQ_PAT.match(val): name = "period" if "p" == m.group(1) else "time" - return {name: int(m.group(2))} + return int(m.group(2)) else: - return {name: val} + return val _FmtIdx: TypeAlias = Callable[[str, str | Any], dict[str, Any]] @@ -157,8 +157,11 @@ def _formatter(index_type: str) -> _FmtIdx: case "duration": return lambda name, key: {name: normalise_freq(key)} case "str": - # custom handling when data matches `SEQ_PAT` - return _atoi + # don't use lambda, can't add type hints + def _atoi_dict(name: str, val: str) -> dict[str, int | str]: + return {name: _atoi(val)} + + return _atoi_dict case "float" | "time_pattern" | "timepattern" | "noop": return lambda name, key: {name: key} case _: # fallback to noop w/ a warning From 8542e3a4e3d2d8e8d2c89b78b33fdee812fa221c Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:32:49 +0200 Subject: [PATCH 17/43] dbmap: check index_name uniqueness, add assertion message --- arrow_expts/spine/dbmap.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 5a5c760..782240c 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -202,15 +202,23 @@ def make_records( # `make_records` available to them, but nothing more. They either # help with some computation, raise a warning, or are helpers to # append to the result. + _msg_assert = ( + "for the type checker: rest of the function expects `json_doc` to be a dict" + ) + + def _uniquify_index_name(default: str) -> str: + assert isinstance(json_doc, dict), _msg_assert + index_name = json_doc.get("index_name", default) + return index_name + f"{len(idx_lvls)}" if index_name in idx_lvls else index_name + def _from_pairs(data: Iterable[Iterable], fmt: _FmtIdx): - assert isinstance(json_doc, dict) - index_name = json_doc.get("index_name", lvlname) + index_name = _uniquify_index_name(lvlname) for key, val in data: _lvls = {**idx_lvls, **fmt(index_name, key)} make_records(val, _lvls, res, lvlname_base=lvlname_base) def _deprecated(var: str, val: Any): - assert isinstance(json_doc, dict) + assert isinstance(json_doc, dict), _msg_assert index_name = json_doc.get("index_name", lvlname) msg = f"{index_name}: {var}={val} is deprecated, handle in model, defaulting to time index from 0001-01-01." warn(msg, DeprecationWarning) @@ -222,8 +230,7 @@ def _time_index(idx: dict, length: int): return low_res_datetime(start=start, freq=freq, periods=length) def _append_arr(arr: Iterable, fmt: _FmtIdx): - assert isinstance(json_doc, dict) - index_name = json_doc.get("index_name", "i") + index_name = _uniquify_index_name("i") for value in arr: res.append({**idx_lvls, **fmt(index_name, value)}) From ab6f76607ae2a0fd2a9e74d2983060bae3633a8e Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:34:17 +0200 Subject: [PATCH 18/43] dbmap: format duration as pandas.DateOffset pandas.DateOffset is similar to relative delta --- arrow_expts/spine/dbmap.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 782240c..4fa83fa 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -21,6 +21,8 @@ def json_loads_ts(json_str: str | bytes): SEQ_PAT = re.compile(r"(t|p)([0-9]+)") # Regex pattern to identify a number encoded as a string FREQ_PAT = re.compile("^[0-9]+$") +# Regex pattern to duration strings +DUR_PAT = re.compile(r"([0-9]+) *(Y|M|W|D|h|min|s)") def normalise_freq(freq: int | str): @@ -107,6 +109,30 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: return date_array_with_frequency +def to_dateoffset(val: str) -> pd.DateOffset: + if (m := DUR_PAT.match(val)) is None: + raise ValueError(f"{val}: bad duration value") + num, freq = m.groups() + match freq: + case "Y": + return pd.DateOffset(years=num) + case "M": + return pd.DateOffset(months=num) + case "W": + return pd.DateOffset(weeks=num) + case "D": + return pd.DateOffset(days=num) + case "h": + return pd.DateOffset(hours=num) + case "min": + return pd.DateOffset(minutes=num) + case "s": + pd.DateOffset(seconds=num) + case _: + # should not get here + raise ValueError(f"{val}: unknown duration") + + def _atoi(val: str) -> int | str: """Convert string to number if it matches `t0001` or `p2001`. @@ -138,9 +164,8 @@ def _formatter(index_type: str) -> _FmtIdx: - "date_time" :: converts value to `datetime` - - "duration" :: converts string to `pandas.Timedelta` compatible - argument; note it still allows for ambiguous units like month or - year. + - "duration" :: converts string to `pandas.DateOffset`; this + allows for ambiguous units like month or year. - "str" :: convert the value to integer if it matches `t0001` or `p2002`, and the name to "time" and "period" respectively; @@ -155,7 +180,7 @@ def _formatter(index_type: str) -> _FmtIdx: case "date_time" | "datetime": return lambda name, key: {name: datetime.fromisoformat(key)} case "duration": - return lambda name, key: {name: normalise_freq(key)} + return lambda name, key: {name: to_dateoffset(normalise_freq(key))} case "str": # don't use lambda, can't add type hints def _atoi_dict(name: str, val: str) -> dict[str, int | str]: From 21d5f959dc25303e371d9aba81a694ee06d855c0 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:38:27 +0200 Subject: [PATCH 19/43] {dbmap,models}.py: time_pattern using simple dataclass --- arrow_expts/schema/models.py | 9 ++++++++- arrow_expts/spine/dbmap.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index 9185b02..152c8bc 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -14,6 +14,7 @@ # from dataclasses import dataclass # from dataclasses import field from datetime import datetime, timedelta +from re import Pattern from typing import Annotated, Literal, TypeAlias import pandas as pd @@ -34,7 +35,13 @@ # FIXME: how to do w/o Pydantic? time_pat_re = r"(Y|M|D|WD|h|m|s)[0-9]+-[0-9]+" -TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)] +@dataclass(frozen=True) +class TimePattern: + pattern: str + re: str | Pattern[str] = time_pat_re + + +# TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)] TimePatterns: TypeAlias = list[TimePattern] NullableIntegers: TypeAlias = list[int | None] diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 4fa83fa..220af9d 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -9,6 +9,7 @@ import pandas as pd import numpy as np +from arrow_expts.schema.models import TimePattern from spinedb_api import DatabaseMapping from spinedb_api.temp_id import TempId @@ -187,7 +188,9 @@ def _atoi_dict(name: str, val: str) -> dict[str, int | str]: return {name: _atoi(val)} return _atoi_dict - case "float" | "time_pattern" | "timepattern" | "noop": + case "time_pattern" | "timepattern": + return lambda name, key: {name: TimePattern(key)} + case "float" | "noop": return lambda name, key: {name: key} case _: # fallback to noop w/ a warning warn(f"{index_type}: unknown type, fallback to noop formatter") @@ -297,6 +300,9 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): _from_pairs(zip(index, data), _formatter("noop")) case {"type": "time_series", "data": [float() | int(), *_] as data}: _append_arr(data, _formatter("noop")) + # time_pattern + case {"type": "time_pattern", "data": dict() as data}: + _from_pairs(data.items(), _formatter("time_pattern")) # arrays case { "type": "array", @@ -306,9 +312,9 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): _append_arr(data, _formatter(value_type)) case {"type": "array", "data": [float() | int(), *_] as data}: _append_arr(data, _formatter("float")) - # date_time | duration | time_pattern + # date_time | duration case { - "type": "date_time" | "duration" | "time_pattern" as data_t, + "type": "date_time" | "duration" as data_t, "data": str() | int() as data, }: _fmt = _formatter(data_t) From 54cbda24be577218dfe8628c1b0225c5a485ca12 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 18:39:08 +0200 Subject: [PATCH 20/43] dbmap: fix handling of array-like time_series --- arrow_expts/spine/dbmap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 220af9d..ec912ce 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -299,7 +299,10 @@ def _append_arr(arr: Iterable, fmt: _FmtIdx): index = _time_index(idx, len(data)) _from_pairs(zip(index, data), _formatter("noop")) case {"type": "time_series", "data": [float() | int(), *_] as data}: - _append_arr(data, _formatter("noop")) + msg = "array-like 'time_series' without time-stamps, relies on 'ignore_year' and 'repeat' implicitly" + warn(msg, DeprecationWarning) + updated = {**json_doc, "index": {"ignore_year": True, "repeat": True}} + make_records(updated, idx_lvls, res, lvlname_base=lvlname_base) # time_pattern case {"type": "time_pattern", "data": dict() as data}: _from_pairs(data.items(), _formatter("time_pattern")) From 232cd7d87be7d6b3a3ad2cb0d76c3463e00a74d2 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 21:17:22 +0200 Subject: [PATCH 21/43] dbmap: bug fix to_dateoffset --- arrow_expts/spine/dbmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index ec912ce..52e4567 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -128,7 +128,7 @@ def to_dateoffset(val: str) -> pd.DateOffset: case "min": return pd.DateOffset(minutes=num) case "s": - pd.DateOffset(seconds=num) + return pd.DateOffset(seconds=num) case _: # should not get here raise ValueError(f"{val}: unknown duration") From 1fac7cd239db88d7e213bde5869c24668744d5de Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 21:20:04 +0200 Subject: [PATCH 22/43] dbmap: remove redundant "renaming" of time/period column --- arrow_expts/spine/dbmap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 52e4567..7df09c3 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -142,7 +142,6 @@ def _atoi(val: str) -> int | str: """ if m := SEQ_PAT.match(val): - name = "period" if "p" == m.group(1) else "time" return int(m.group(2)) else: return val From 0c054975f35a59df88d3a4ebfcdde7e6bb317551 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 10 Apr 2025 21:33:31 +0200 Subject: [PATCH 23/43] tests: sample JSON file was incorrect Arrays in the absense of "value_type" default to "float" (as per the docs), here the data is text string, not convertible to a number. --- tests/json/array.strings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/json/array.strings.json b/tests/json/array.strings.json index fa93e6f..7990ce6 100644 --- a/tests/json/array.strings.json +++ b/tests/json/array.strings.json @@ -1,5 +1,6 @@ { "type": "array", "data": ["one", "two"], + "value_type": "str", "index_name": "step" } From d39af2df9f32a4a2828c85110ebfcaeb63dec91c Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:11:32 +0200 Subject: [PATCH 24/43] dbmap: two bug fixes - minute is abbreviated as min - convert freq to int before passing to DateOffset --- arrow_expts/spine/dbmap.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 7df09c3..bccc1f8 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -35,10 +35,10 @@ def normalise_freq(freq: int | str): """ if isinstance(freq, int): - return str(freq) + "m" + return str(freq) + "min" if FREQ_PAT.match(freq): # If frequency is an integer, the implied unit is "minutes" - return freq + "m" + return freq + "min" # not very robust yet return ( freq.replace("years", "Y") @@ -113,7 +113,8 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: def to_dateoffset(val: str) -> pd.DateOffset: if (m := DUR_PAT.match(val)) is None: raise ValueError(f"{val}: bad duration value") - num, freq = m.groups() + num_str, freq = m.groups() + num = int(num_str) match freq: case "Y": return pd.DateOffset(years=num) From a583807331aa4a9931af81d1b326fb7bfc76d456 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:15:05 +0200 Subject: [PATCH 25/43] models.py: simplify bytes array --- arrow_expts/schema/models.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index 152c8bc..910aac3 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -28,7 +28,7 @@ Integers: TypeAlias = list[int] Strings: TypeAlias = list[str] Booleans: TypeAlias = list[bool] -BytesList: TypeAlias = list[bytes] +BytesList: TypeAlias = list[bytes] # array of bytes to support mixed types Datetimes: TypeAlias = list[datetime] Timedeltas: TypeAlias = list[timedelta] @@ -48,6 +48,7 @@ class TimePattern: NullableFloats: TypeAlias = list[float | None] NullableStrings: TypeAlias = list[str | None] NullableBooleans: TypeAlias = list[bool | None] +NullableBytesList: TypeAlias = list[bytes | None] NullableDatetimes: TypeAlias = list[datetime | None] NullableTimedeltas: TypeAlias = list[timedelta | None] NullableTimePatterns: TypeAlias = list[TimePattern | None] @@ -68,6 +69,7 @@ class TimePattern: | NullableStrings | NullableFloats | NullableBooleans + | NullableBytesList | NullableDatetimes | NullableTimedeltas | NullableTimePatterns @@ -75,7 +77,14 @@ class TimePattern: ValueTypeNames: TypeAlias = Literal[ - "string", "integer", "number", "boolean", "date-time", "duration", "time-pattern" + "string", + "integer", + "number", + "boolean", + "bytes", + "date-time", + "duration", + "time-pattern", ] IndexValueTypeNames: TypeAlias = Literal[ "string", "integer", "date-time", "duration", "time-pattern" @@ -198,27 +207,11 @@ class Array(_TypeInferMixin): type: Literal["array"] = "array" -@dataclass(frozen=True) -class BytesArray(_TypeInferMixin): - """Array of bytes to store mixed types""" - - name: str - values: BytesList - value_type: ValueTypeNames = field(init=False) - type: Literal["bytes_array"] = "bytes_array" - - # NOTE: To add run-length encoding to the schema, add it to the # following type union following which, we need to implement a # converter to an Arrow array type Table: TypeAlias = list[ - RunEndIndex - | DictEncodedIndex - | ArrayIndex - | RunEndArray - | DictEncodedArray - | Array - | BytesArray + RunEndIndex | DictEncodedIndex | ArrayIndex | RunEndArray | DictEncodedArray | Array ] From 1edb2a81a56d9423b52c23f34c8e44ab4f611e5d Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:16:54 +0200 Subject: [PATCH 26/43] models.py: add more type maps --- arrow_expts/schema/models.py | 14 ++++++++++++++ pyproject.toml | 1 + 2 files changed, 15 insertions(+) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index 910aac3..5d68111 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "python-dateutil", # "pandas>=2", # "pydantic>=2", # ] @@ -17,6 +18,8 @@ from re import Pattern from typing import Annotated, Literal, TypeAlias +from dateutil.relativedelta import relativedelta +import numpy as np import pandas as pd from pydantic import RootModel from pydantic.dataclasses import dataclass @@ -93,12 +96,23 @@ class TimePattern: type_map: dict[type, ValueTypeNames] = { str: "string", int: "integer", + np.int8: "integer", + np.int16: "integer", + np.int32: "integer", + np.int64: "integer", float: "number", + np.float16: "number", + np.float32: "number", + np.float64: "number", + np.float128: "number", bool: "boolean", + np.bool: "boolean", datetime: "date-time", pd.Timestamp: "date-time", timedelta: "duration", pd.Timedelta: "duration", + relativedelta: "duration", + pd.DateOffset: "duration", TimePattern: "time-pattern", bytes: "string", } diff --git a/pyproject.toml b/pyproject.toml index e3dc54a..158f44c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "pyarrow", "pydantic", "pytest", + "python-dateutil", "rich", "spinedb-api", "sqlalchemy", From 6b34e62a0d4798d1e3c507d1e04d08a7b8d2ac0e Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:19:19 +0200 Subject: [PATCH 27/43] models: add alternate TimePattern w/o Pydantic Refactor the module w/ conditional imports to use Pydantic for schema generation (invoked as `python -m models.py json-blob-schema.json`), but use plain Python definitions when used as a module. Update the JSON schema --- arrow_expts/schema/json-blob-schema.json | 45 ++++++++++++++++++++++++ arrow_expts/schema/models.py | 40 +++++++++++++-------- 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/arrow_expts/schema/json-blob-schema.json b/arrow_expts/schema/json-blob-schema.json index 672b1a6..222d14a 100644 --- a/arrow_expts/schema/json-blob-schema.json +++ b/arrow_expts/schema/json-blob-schema.json @@ -61,6 +61,20 @@ }, "type": "array" }, + { + "items": { + "anyOf": [ + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, { "items": { "anyOf": [ @@ -112,6 +126,7 @@ "integer", "number", "boolean", + "bytes", "date-time", "duration", "time-pattern" @@ -280,6 +295,20 @@ }, "type": "array" }, + { + "items": { + "anyOf": [ + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, { "items": { "anyOf": [ @@ -331,6 +360,7 @@ "integer", "number", "boolean", + "bytes", "date-time", "duration", "time-pattern" @@ -501,6 +531,20 @@ }, "type": "array" }, + { + "items": { + "anyOf": [ + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, { "items": { "anyOf": [ @@ -552,6 +596,7 @@ "integer", "number", "boolean", + "bytes", "date-time", "duration", "time-pattern" diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index 5d68111..ce5588a 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -12,19 +12,19 @@ """ -# from dataclasses import dataclass -# from dataclasses import field from datetime import datetime, timedelta -from re import Pattern from typing import Annotated, Literal, TypeAlias from dateutil.relativedelta import relativedelta import numpy as np import pandas as pd -from pydantic import RootModel -from pydantic.dataclasses import dataclass -from pydantic.dataclasses import Field as field -from pydantic.types import StringConstraints + +if __name__ == "__main__": + from pydantic.dataclasses import dataclass + from pydantic.dataclasses import Field as field +else: + from dataclasses import dataclass + from dataclasses import field Floats: TypeAlias = list[float] @@ -38,15 +38,24 @@ # FIXME: how to do w/o Pydantic? time_pat_re = r"(Y|M|D|WD|h|m|s)[0-9]+-[0-9]+" -@dataclass(frozen=True) -class TimePattern: - pattern: str - re: str | Pattern[str] = time_pat_re + +# generate schema w/ Pydantic by running as a script +if __name__ == "__main__": + from pydantic.types import StringConstraints + + TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)] +else: + from re import Pattern + + @dataclass(frozen=True) + class TimePattern: + pattern: str + re: str | Pattern[str] = time_pat_re -# TimePattern: TypeAlias = Annotated[str, StringConstraints(pattern=time_pat_re)] TimePatterns: TypeAlias = list[TimePattern] +# nullable variant of arrays NullableIntegers: TypeAlias = list[int | None] NullableFloats: TypeAlias = list[float | None] NullableStrings: TypeAlias = list[str | None] @@ -56,6 +65,7 @@ class TimePattern: NullableTimedeltas: TypeAlias = list[timedelta | None] NullableTimePatterns: TypeAlias = list[TimePattern | None] +# sets of types used to define array schemas below IndexTypes: TypeAlias = Integers | Strings | Datetimes | Timedeltas | TimePatterns ValueTypes: TypeAlias = ( Integers @@ -78,7 +88,7 @@ class TimePattern: | NullableTimePatterns ) - +# names of types used in the schema ValueTypeNames: TypeAlias = Literal[ "string", "integer", @@ -223,7 +233,7 @@ class Array(_TypeInferMixin): # NOTE: To add run-length encoding to the schema, add it to the # following type union following which, we need to implement a -# converter to an Arrow array type +# converter to a compatible pyarrow array type Table: TypeAlias = list[ RunEndIndex | DictEncodedIndex | ArrayIndex | RunEndArray | DictEncodedArray | Array ] @@ -234,6 +244,8 @@ class Array(_TypeInferMixin): import json from pathlib import Path + from pydantic import RootModel + parser = ArgumentParser(__doc__) parser.add_argument("json_file", help="Path of JSON schema file to write") opts = parser.parse_args() From b193d741a126372cd2c7eb09e45c584e08f6565a Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:22:24 +0200 Subject: [PATCH 28/43] dbmap: rename internal functions as `_` --- arrow_expts/spine/dbmap.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index bccc1f8..f831080 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -26,7 +26,7 @@ def json_loads_ts(json_str: str | bytes): DUR_PAT = re.compile(r"([0-9]+) *(Y|M|W|D|h|min|s)") -def normalise_freq(freq: int | str): +def _normalise_freq(freq: int | str): """Normalise integer/string to frequency. The frequency value is as understood by `pandas.Timedelta`. Note @@ -58,7 +58,7 @@ def normalise_freq(freq: int | str): ) -to_numpy = { +_to_numpy_time_units = { "Y": "Y", "M": "M", "W": "W", @@ -69,7 +69,7 @@ def normalise_freq(freq: int | str): } -def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: +def _low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: """Create pd.DatetimeIndex with lower time resolution. The default resolution of pd.date_time is [ns], which puts @@ -97,7 +97,7 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: start_date_np = np.datetime64(start, "s") # print(f"start_date_np: {start_date_np}") # print(to_numpy[unit]) - freq_np = np.timedelta64(int(number_str), to_numpy[unit]) + freq_np = np.timedelta64(int(number_str), _to_numpy_time_units[unit]) # print(f"freq_np: {freq_np}") freq_pd = pd.Timedelta(freq_np) # print(f"freq_pd: {freq_pd}") @@ -110,7 +110,7 @@ def low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: return date_array_with_frequency -def to_dateoffset(val: str) -> pd.DateOffset: +def _to_dateoffset(val: str) -> pd.DateOffset: if (m := DUR_PAT.match(val)) is None: raise ValueError(f"{val}: bad duration value") num_str, freq = m.groups() @@ -181,7 +181,7 @@ def _formatter(index_type: str) -> _FmtIdx: case "date_time" | "datetime": return lambda name, key: {name: datetime.fromisoformat(key)} case "duration": - return lambda name, key: {name: to_dateoffset(normalise_freq(key))} + return lambda name, key: {name: _to_dateoffset(_normalise_freq(key))} case "str": # don't use lambda, can't add type hints def _atoi_dict(name: str, val: str) -> dict[str, int | str]: @@ -254,8 +254,8 @@ def _deprecated(var: str, val: Any): def _time_index(idx: dict, length: int): start = idx.get("start", "0001-01-01T00:00:00") resolution = idx.get("resolution", "1h") - freq = normalise_freq(resolution) - return low_res_datetime(start=start, freq=freq, periods=length) + freq = _normalise_freq(resolution) + return _low_res_datetime(start=start, freq=freq, periods=length) def _append_arr(arr: Iterable, fmt: _FmtIdx): index_name = _uniquify_index_name("i") From 2b5d06c253d5e001c3c25400d5162666c328def9 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:52:01 +0200 Subject: [PATCH 29/43] tests/test_dbmap: add and update tests --- tests/test_dbmap.py | 85 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/tests/test_dbmap.py b/tests/test_dbmap.py index 1cdf7a1..7126a77 100644 --- a/tests/test_dbmap.py +++ b/tests/test_dbmap.py @@ -1,24 +1,95 @@ +from datetime import datetime +import json +from typing import Callable + import numpy as np import pandas as pd +import pytest + +from arrow_expts.schema.models import TimePattern +from arrow_expts.spine.dbmap import ( + _atoi, + _formatter, + _low_res_datetime, + _normalise_freq, + _to_dateoffset, + make_records, +) -from arrow_expts.spine.dbmap import normalise_freq, low_res_datetime +from .conftest import JSONDIR def test_filter_frequencies(): - assert normalise_freq("5 years") == "5 Y" - assert normalise_freq("5 year") == "5 Y" - assert normalise_freq("3s") == "3s" - assert normalise_freq("60") == "60m" + assert _normalise_freq("5 years") == "5 Y" + assert _normalise_freq("5 year") == "5 Y" + assert _normalise_freq("3s") == "3s" + assert _normalise_freq("60") == "60min" def test_low_res_datetime(): target = pd.date_range(start="2001-01-01", freq="7D", periods=5) - low_res = low_res_datetime(start="2001-01-01", freq="1W", periods=5) + low_res = _low_res_datetime(start="2001-01-01", freq="1W", periods=5) assert low_res.dtype == "datetime64[s]" assert target.freq == low_res.freq for date_target, date_low_res in zip(target, low_res): assert date_target == date_low_res - res = low_res_datetime(start="0001-01-01", freq="1s", periods=1) + res = _low_res_datetime(start="0001-01-01", freq="1s", periods=1) assert res[0] == pd.Timestamp("0001-01-01") + + +@pytest.mark.parametrize("part", ["durations", "numbers", "strings"]) +def test_arrays(part: str): + data = json.loads((JSONDIR / f"array.{part}.json").read_text()) + fmt = _formatter(data.get("value_type", "float")) + index_name = data.get("index_name", "i") + expect = [fmt(index_name, i) for i in data["data"]] + assert expect == make_records(data, {}, []) + + +def fmt_durations(val: str | int): + return _to_dateoffset(_normalise_freq(val)) + + +@pytest.mark.parametrize( + "type_,value,fmt", + [ + ("date_time", "2019-06-01T22:15:00+01:00", datetime.fromisoformat), + ("duration", "1h", fmt_durations), + ("duration", 60, fmt_durations), + ("duration", "1 hour", fmt_durations), + ], +) +def test_values(type_: str, value, fmt: Callable): + res = make_records({"type": type_, "data": value}, {}, []) + assert res == [{"value": fmt(value)}] + + +def test_time_pattern(): + data = json.loads((JSONDIR / "time-pattern.json").read_text()) + index_name = data.get("index_name", "default0") + expect = [{index_name: TimePattern(k), "value": v} for k, v in data["data"].items()] + res = make_records(data, {}, []) + assert res == expect + + +@pytest.mark.parametrize( + "part, interval", + [ + ("dict", "30 min"), + ("one-column-array-custom-indices", "30 min"), + ("one-column-array", "1 h"), + ("two-column-array", "30 min"), + ("two-column-array-named-indices", "30 min"), + ], +) +def test_time_series(part: str, interval: str): + data = json.loads((JSONDIR / f"time-series.{part}.json").read_text()) + res = make_records(data, {}, []) + + # index column name matches + index_name = data.get("index_name", "default0") + assert index_name in res[0] + assert isinstance(res[0][index_name], datetime) + assert res[1][index_name] - res[0][index_name] == pd.Timedelta(interval) From ade650964398fb416701456300865e8349bce797 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Fri, 11 Apr 2025 17:52:27 +0200 Subject: [PATCH 30/43] tests/test_integration: refactor for numpy array comparison --- tests/test_integration.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 163aa91..6833345 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,5 +1,7 @@ +from dataclasses import astuple import json +import numpy as np import pytest from arrow_expts.schema.models import Array @@ -8,6 +10,13 @@ from .conftest import JSONDIR +def np_eq(i, j): + if isinstance(cmp := (i == j), np.ndarray): + return cmp.all() + else: + return cmp + + @pytest.mark.parametrize("fname,arr", [("array.numbers.json", [2.3, 23.0, 5.0])]) def test_json_to_tbl(fname, arr): input_json = JSONDIR / fname @@ -16,5 +25,10 @@ def test_json_to_tbl(fname, arr): exp = Array(name="i", values=arr) df = to_df(data) - assert series_to_col(df[exp.name]) == exp - assert to_tables(df) == [exp] + res = series_to_col(df[exp.name]) + for i, j in zip(astuple(res), astuple(exp)): + assert np_eq(i, j) + + tbl = to_tables(df) + assert len(tbl) == 1 + assert isinstance(tbl[0], Array) From 91d920b0c0c0a8d9b1430a1527b61c04b45510bd Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 19:20:26 +0200 Subject: [PATCH 31/43] dbmap: refactor for simpler & more consistent use of datetime --- arrow_expts/spine/dbmap.py | 18 ++++-------------- tests/test_dbmap.py | 5 ++--- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index f831080..9984c33 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -1,5 +1,4 @@ from argparse import ArgumentParser -from datetime import datetime import json import re from typing import cast, Any, Callable, Iterable, TypeAlias @@ -85,22 +84,14 @@ def _low_res_datetime(start: str, freq: str, periods: int) -> pd.DatetimeIndex: `pd.Timestamp.max`. """ - if re_match := re.search(r"^([0-9]+) *(.*)$", freq): - period_parts = re_match.groups() + if re_match := DUR_PAT.match(_normalise_freq(freq)): + number_str, unit = re_match.groups() else: raise ValueError(f"invalid frequency: {freq!r}") - if len(period_parts) != 2: - raise ValueError(f"invalid frequency: {freq!r}") - - number_str, unit = period_parts start_date_np = np.datetime64(start, "s") - # print(f"start_date_np: {start_date_np}") - # print(to_numpy[unit]) freq_np = np.timedelta64(int(number_str), _to_numpy_time_units[unit]) - # print(f"freq_np: {freq_np}") freq_pd = pd.Timedelta(freq_np) - # print(f"freq_pd: {freq_pd}") date_array = np.arange(start_date_np, start_date_np + periods * freq_np, freq_np) date_array_with_frequency = pd.DatetimeIndex( @@ -179,7 +170,7 @@ def _formatter(index_type: str) -> _FmtIdx: """ match index_type: case "date_time" | "datetime": - return lambda name, key: {name: datetime.fromisoformat(key)} + return lambda name, key: {name: pd.Timestamp(key)} case "duration": return lambda name, key: {name: _to_dateoffset(_normalise_freq(key))} case "str": @@ -254,8 +245,7 @@ def _deprecated(var: str, val: Any): def _time_index(idx: dict, length: int): start = idx.get("start", "0001-01-01T00:00:00") resolution = idx.get("resolution", "1h") - freq = _normalise_freq(resolution) - return _low_res_datetime(start=start, freq=freq, periods=length) + return _low_res_datetime(start=start, freq=resolution, periods=length) def _append_arr(arr: Iterable, fmt: _FmtIdx): index_name = _uniquify_index_name("i") diff --git a/tests/test_dbmap.py b/tests/test_dbmap.py index 7126a77..e3e5f21 100644 --- a/tests/test_dbmap.py +++ b/tests/test_dbmap.py @@ -1,4 +1,3 @@ -from datetime import datetime import json from typing import Callable @@ -55,7 +54,7 @@ def fmt_durations(val: str | int): @pytest.mark.parametrize( "type_,value,fmt", [ - ("date_time", "2019-06-01T22:15:00+01:00", datetime.fromisoformat), + ("date_time", "2019-06-01T22:15:00+01:00", pd.Timestamp), ("duration", "1h", fmt_durations), ("duration", 60, fmt_durations), ("duration", "1 hour", fmt_durations), @@ -91,5 +90,5 @@ def test_time_series(part: str, interval: str): # index column name matches index_name = data.get("index_name", "default0") assert index_name in res[0] - assert isinstance(res[0][index_name], datetime) + assert isinstance(res[0][index_name], pd.Timestamp) assert res[1][index_name] - res[0][index_name] == pd.Timedelta(interval) From 3bc1325ded33e277b79bfafddd1a109bad4b1127 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 19:23:30 +0200 Subject: [PATCH 32/43] tests: add tests for maps --- tests/test_dbmap.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_dbmap.py b/tests/test_dbmap.py index e3e5f21..3c4151e 100644 --- a/tests/test_dbmap.py +++ b/tests/test_dbmap.py @@ -1,4 +1,5 @@ import json +from types import NoneType from typing import Callable import numpy as np @@ -92,3 +93,36 @@ def test_time_series(part: str, interval: str): assert index_name in res[0] assert isinstance(res[0][index_name], pd.Timestamp) assert res[1][index_name] - res[0][index_name] == pd.Timedelta(interval) + + +@pytest.mark.parametrize( + "part, lvls, types", + [ + ( + "dictionary", + ["default0", "default1", "value"], + [pd.Timestamp, pd.DateOffset, (float, int)], + ), + ( + "stochastic-time-series-antti", + ["stochastic_scenario", "analysis_time", "default2", "value"], + [str, pd.Timestamp, pd.Timestamp, (float, int)], + ), + ( + "stochastic-time-series", + ["Forecast time", "Target time", "Stochastic scenario", "value"], + [pd.Timestamp, pd.Timestamp, int, (float, int)], + ), + ("two-column-array", ["default0", "value"], [str, (float, int)]), + ], +) +def test_map(part: str, lvls: list[str], types: list[type]): + data = json.loads((JSONDIR / f"map.{part}.json").read_text()) + res = make_records(data, {}, []) + + ncols = len(lvls) + assert all(len(i) <= ncols for i in res) + # fallback to `NoneType` to account for missing values + assert all( + isinstance(i.get(k), (t, NoneType)) for i in res for k, t in zip(lvls, types) + ) From c0e0972a3d4dfac7894133f85723d246ce691275 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 19:32:45 +0200 Subject: [PATCH 33/43] dbmap: anchor SEQ_PAT regex, add tests --- arrow_expts/spine/dbmap.py | 2 +- tests/test_dbmap.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/arrow_expts/spine/dbmap.py b/arrow_expts/spine/dbmap.py index 9984c33..8053dc8 100644 --- a/arrow_expts/spine/dbmap.py +++ b/arrow_expts/spine/dbmap.py @@ -18,7 +18,7 @@ def json_loads_ts(json_str: str | bytes): # Regex pattern to indentify numerical sequences encoded as string -SEQ_PAT = re.compile(r"(t|p)([0-9]+)") +SEQ_PAT = re.compile(r"^(t|p)([0-9]+)$") # Regex pattern to identify a number encoded as a string FREQ_PAT = re.compile("^[0-9]+$") # Regex pattern to duration strings diff --git a/tests/test_dbmap.py b/tests/test_dbmap.py index 3c4151e..b79202d 100644 --- a/tests/test_dbmap.py +++ b/tests/test_dbmap.py @@ -39,6 +39,14 @@ def test_low_res_datetime(): assert res[0] == pd.Timestamp("0001-01-01") +@pytest.mark.parametrize( + "val, expect", + [("foo", "foo"), ("t0012", 12), ("p2023", 2023), ("t0012.1", "t0012.1")], +) +def test_atoi(val: str, expect: int | str): + assert expect == _atoi(val) + + @pytest.mark.parametrize("part", ["durations", "numbers", "strings"]) def test_arrays(part: str): data = json.loads((JSONDIR / f"array.{part}.json").read_text()) From 0f904d1b6f01534454e1b4d9c8dd05b7f5154c0a Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:29:07 +0200 Subject: [PATCH 34/43] Add CI --- .github/workflows/unittests.yml | 102 ++++++++++++++++++++++++++++++++ pyproject.toml | 5 ++ 2 files changed, 107 insertions(+) create mode 100644 .github/workflows/unittests.yml diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml new file mode 100644 index 0000000..ca8c11f --- /dev/null +++ b/.github/workflows/unittests.yml @@ -0,0 +1,102 @@ +name: Unit tests + +on: + push: + paths-ignore: + - "docs/**" + - "*.md" + pull_request: + paths-ignore: + - "docs/**" + - "*.md" + +jobs: + format: + if: | + (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && + (!contains(github.event.head_commit.message, 'skip ci')) + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black toml + - name: Format source w/ black + # fail when file(s) would be formatted + run: black --check arrow_expts tests + + lint: + if: | + (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && + (!contains(github.event.head_commit.message, 'skip ci')) + + strategy: + matrix: + pyver: ["py310", "py313"] + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Ruff with Python target version ${{ matrix.pyver }} + uses: chartboost/ruff-action@v1 + with: + src: arrow_expts + args: "check --target-version ${{ matrix.pyver }}" + + test: + if: | + (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && + (!contains(github.event.head_commit.message, 'skip ci')) + needs: [format, lint] + + strategy: + matrix: + python-version: ["3.10", "3.13", "3.14"] + os: [ubuntu-latest, windows-latest, macos-latest] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + allow-prereleases: true + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install . + - name: Run pytest + run: | + pytest -vv tests + + type-hints: + if: | + (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && + (!contains(github.event.head_commit.message, 'skip ci')) + + strategy: + matrix: + python-version: ["3.10", "3.13", "3.14"] + + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + allow-prereleases: true + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mypy{,_extensions} + - name: Type check w/ mypy + run: echo y | mypy --install-types --check-untyped-defs --pretty arrow_expts diff --git a/pyproject.toml b/pyproject.toml index 158f44c..b360043 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,11 @@ dependencies = [ "spinedb-api", "sqlalchemy", ] +[project.optional-dependencies] +dev = [ + "mypy", + "pytest", +] [tool.hatch.metadata.source] include = ["arrow_expts"] From 965434138faf7eac7c0c1b560dffe09d06f46464 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:32:36 +0200 Subject: [PATCH 35/43] fix linting --- arrow_expts/schema/reencode.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index 50d7b15..e38c787 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -18,10 +18,9 @@ from typing import overload import pandas as pd -import pyarrow as pa from pydantic import RootModel -from rich.pretty import pprint +from rich.pretty import pprint # noqa: F401, keep for debugging from ..spine.dbmap import make_records from .models import ( From 8050c0a2997de77f81c492a56b56c3def9971757 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:35:27 +0200 Subject: [PATCH 36/43] fix formatting --- arrow_expts/schema/models.py | 4 +--- arrow_expts/schema/reencode.py | 4 +--- arrow_expts/utils.py | 36 ++++++++++++++++------------------ 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index ce5588a..1f70c95 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -8,9 +8,7 @@ # ] # /// -"""Write JSON schema for JSON blob in SpineDB - -""" +"""Write JSON schema for JSON blob in SpineDB""" from datetime import datetime, timedelta from typing import Annotated, Literal, TypeAlias diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index e38c787..a459841 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -8,9 +8,7 @@ # ] # /// -"""Reencode old map type JSON to new table/tables type JSON - -""" +"""Reencode old map type JSON to new table/tables type JSON""" from datetime import datetime, timedelta import json diff --git a/arrow_expts/utils.py b/arrow_expts/utils.py index db22730..8f1b69e 100644 --- a/arrow_expts/utils.py +++ b/arrow_expts/utils.py @@ -7,9 +7,9 @@ from typing import Any, Callable -def execute_on_sqlite(database_uri: str, - my_function: Callable, - *args: Any, **kwargs: Any) -> Any: +def execute_on_sqlite( + database_uri: str, my_function: Callable, *args: Any, **kwargs: Any +) -> Any: """Execute a function on an SQlite database. Returns: @@ -21,10 +21,13 @@ def execute_on_sqlite(database_uri: str, connection.commit() return result -def _insert_data_into_table(cursor: adbc_driver_manager.dbapi.Cursor, - table_name: str, - data: pa.lib.Table, - mode: str) -> int: + +def _insert_data_into_table( + cursor: adbc_driver_manager.dbapi.Cursor, + table_name: str, + data: pa.lib.Table, + mode: str, +) -> int: """Create a table and fill it with data or append data to an existing one. Args: @@ -44,11 +47,9 @@ def _insert_data_into_table(cursor: adbc_driver_manager.dbapi.Cursor, return result - -def write_data_to_db(database_uri: str, - table_name: str, - data: pa.lib.Table, - mode: str) -> int: +def write_data_to_db( + database_uri: str, table_name: str, data: pa.lib.Table, mode: str +) -> int: """Create a table and fill it with data or append data to an existing one. Args: @@ -60,16 +61,13 @@ def write_data_to_db(database_uri: str, Returns: Number of rows inserted """ - rows_inserted: int = execute_on_sqlite(database_uri, - _insert_data_into_table, - table_name, - data, - mode) + rows_inserted: int = execute_on_sqlite( + database_uri, _insert_data_into_table, table_name, data, mode + ) return rows_inserted -def database_equality(uri_database_1: str, - uri_database_2: str) -> bool: +def database_equality(uri_database_1: str, uri_database_2: str) -> bool: """Compare equality of two SQlite databases.""" connection: adbc_driver_manager.dbapi.Connection From 356dd1ee749f8559c7d6c47b80229d86a028aeef Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:41:52 +0200 Subject: [PATCH 37/43] models: minor platform dependent fix --- arrow_expts/schema/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow_expts/schema/models.py b/arrow_expts/schema/models.py index 1f70c95..e68630b 100755 --- a/arrow_expts/schema/models.py +++ b/arrow_expts/schema/models.py @@ -112,7 +112,7 @@ class TimePattern: np.float16: "number", np.float32: "number", np.float64: "number", - np.float128: "number", + # np.float128: "number", # not available on macos bool: "boolean", np.bool: "boolean", datetime: "date-time", From 5a92848d64930cf158ba135e4585921e254c4383 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:43:48 +0200 Subject: [PATCH 38/43] tests: add missing conftest.py --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1781ccf --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,3 @@ +from pathlib import Path + +JSONDIR = Path("tests/json") From 9417baeac246c497d1b5fc7f9af09d44f606601e Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:50:16 +0200 Subject: [PATCH 39/43] ci: run prerelease only on linux --- .github/workflows/unittests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index ca8c11f..25f68c1 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -57,8 +57,11 @@ jobs: strategy: matrix: - python-version: ["3.10", "3.13", "3.14"] + python-version: ["3.10", "3.13"] os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + python-version: "3.14" runs-on: ${{ matrix.os }} steps: From 60653923d62d24bdb90c2585da280fef7815ad6f Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sat, 12 Apr 2025 20:57:46 +0200 Subject: [PATCH 40/43] ci: disable testing on prerelease because not all wheels aren't avl --- .github/workflows/unittests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 25f68c1..9680e66 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -59,9 +59,9 @@ jobs: matrix: python-version: ["3.10", "3.13"] os: [ubuntu-latest, windows-latest, macos-latest] - include: - - os: ubuntu-latest - python-version: "3.14" + # include: + # - python-version: "3.14" + # os: ubuntu-latest runs-on: ${{ matrix.os }} steps: From 92febb74953ded5235647c481efa8f0ac78163e2 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 1 May 2025 14:22:28 +0200 Subject: [PATCH 41/43] reencode: fix types --- arrow_expts/schema/reencode.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index a459841..064b45c 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -58,7 +58,7 @@ def rl_encode(arr: Array) -> RunLengthArray: ... def rl_encode(arr: ArrayIndex) -> RunLengthIndex: ... -def rl_encode(arr: ArrayIndex | Array) -> RunLengthIndex | RunLengthArray: +def rl_encode(arr): last = SENTINEL values, run_len = [], [] for val in arr.values: @@ -68,7 +68,10 @@ def rl_encode(arr: ArrayIndex | Array) -> RunLengthIndex | RunLengthArray: last = val else: run_len[-1] += 1 - return RunLengthIndex(name=arr.name, values=values, run_len=run_len) + if isinstance(arr, ArrayIndex): + return RunLengthIndex(name=arr.name, values=values, run_len=run_len) + else: + return RunLengthArray(name=arr.name, values=values, run_len=run_len) @overload @@ -79,7 +82,7 @@ def re_encode(arr: Array) -> RunEndArray: ... def re_encode(arr: ArrayIndex) -> RunEndIndex: ... -def re_encode(arr: ArrayIndex | Array) -> RunEndIndex | RunEndArray: +def re_encode(arr): last = SENTINEL values, run_end = [], [] for idx, val in enumerate(arr.values, start=1): @@ -89,7 +92,10 @@ def re_encode(arr: ArrayIndex | Array) -> RunEndIndex | RunEndArray: else: run_end[-1] = idx last = val - return RunEndIndex(name=arr.name, values=values, run_end=run_end) + if isinstance(arr, ArrayIndex): + return RunEndIndex(name=arr.name, values=values, run_end=run_end) + else: + return RunEndArray(name=arr.name, values=values, run_end=run_end) @overload @@ -100,11 +106,14 @@ def de_encode(arr: Array) -> DictEncodedArray: ... def de_encode(arr: ArrayIndex) -> DictEncodedIndex: ... -def de_encode(arr: ArrayIndex | Array) -> DictEncodedIndex | DictEncodedArray: +def de_encode(arr): # not using list(set(...)) to preserve order values = list(dict.fromkeys(arr.values)) indices = list(map(values.index, arr.values)) - return DictEncodedIndex(name=arr.name, values=values, indices=indices) + if isinstance(arr, ArrayIndex): + return DictEncodedIndex(name=arr.name, values=values, indices=indices) + else: + return DictEncodedArray(name=arr.name, values=values, indices=indices) def series_to_col( From 921dbc8840d6d7b95776278aae5466792d943d7e Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 1 May 2025 14:57:27 +0200 Subject: [PATCH 42/43] reencode: more precise types and robust pattern matching --- arrow_expts/schema/reencode.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/arrow_expts/schema/reencode.py b/arrow_expts/schema/reencode.py index 064b45c..43901ff 100755 --- a/arrow_expts/schema/reencode.py +++ b/arrow_expts/schema/reencode.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import overload +import numpy as np import pandas as pd from pydantic import RootModel @@ -120,8 +121,7 @@ def series_to_col( col: pd.Series, ) -> ArrayIndex | DictEncodedIndex | Array | DictEncodedArray: match col.name, col.dtype.type: - case "value", t if issubclass(t, str) or t is object: - print(f"type: {t}, value: {col.iloc[:3]}") + case "value", t if issubclass(t, str) or t in (object, np.object_): col = col.astype("category") return DictEncodedArray( name=col.name, @@ -129,19 +129,18 @@ def series_to_col( indices=col.cat.codes, ) case "value", t if issubclass(t, int): - return Array(name=col.name, values=col.values) + return Array(name=col.name, values=col.to_list()) case _, t if issubclass(t, (bool, float, bytes)): - return Array(name=col.name, values=col.values) - case _, t if issubclass(t, str) or t is object: - print(f"idx_type: {t}, value: {col.iloc[:3]}") + return Array(name=col.name, values=col.to_list()) + case _, t if issubclass(t, str) or t in (object, np.object_): col = col.astype("category") return DictEncodedIndex( name=col.name, - values=col.cat.categories, - indices=col.cat.codes, + values=col.cat.categories.to_list(), + indices=col.cat.codes.to_list(), ) case _, t if issubclass(t, (int, datetime, timedelta)) or t is object: - return ArrayIndex(name=col.name, values=col.values) + return ArrayIndex(name=col.name, values=col.to_list()) case n, t: raise NotImplementedError(f"{n}: unknown type {t}") From 7f22c5ef518aa569b27929b2c569de2075469eb0 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Thu, 1 May 2025 14:58:03 +0200 Subject: [PATCH 43/43] Fix integration tests (only for "numbers") --- tests/test_integration.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6833345..323e3fe 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,6 +2,7 @@ import json import numpy as np +import pandas as pd import pytest from arrow_expts.schema.models import Array @@ -11,18 +12,35 @@ def np_eq(i, j): - if isinstance(cmp := (i == j), np.ndarray): + if isinstance(cmp := (i == j), (np.ndarray, pd.Series, pd.DataFrame)): return cmp.all() else: return cmp -@pytest.mark.parametrize("fname,arr", [("array.numbers.json", [2.3, 23.0, 5.0])]) -def test_json_to_tbl(fname, arr): - input_json = JSONDIR / fname +@pytest.mark.parametrize("part", ["numbers"]) +def test_series_to_col(part): + input_json = JSONDIR / f"array.{part}.json" data = json.loads(input_json.read_text()) - exp = Array(name="i", values=arr) + exp = Array(name=data.get("index_name", "i"), values=data["data"]) + df = to_df(data) + + res = series_to_col(df[exp.name]) + for i, j in zip(astuple(res), astuple(exp)): + assert np_eq(i, j) + + tbl = to_tables(df) + assert len(tbl) == 1 + assert isinstance(tbl[0], Array) + + +@pytest.mark.parametrize("part", ["numbers"]) +def test_to_tables(part): + input_json = JSONDIR / f"array.{part}.json" + data = json.loads(input_json.read_text()) + + exp = Array(name=data.get("index_name", "i"), values=data["data"]) df = to_df(data) res = series_to_col(df[exp.name])