Skip to content

fix: rename metaculus _create_resolution_frame to _build_resolution frame and fix return annotation #209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nikbpetrov wants to merge 1 commit into from
Closed

fix: rename metaculus _create_resolution_frame to _build_resolution frame and fix return annotation #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/sources/metaculus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def update(
dfq.at[index, "forecast_horizons"] = "N/A"

# Build resolution file
df_res = self._create_resolution_file(dfq, index, market)
df_res = self._build_resolution_file(dfq, index, market)
if df_res is not None:
resolution_files[str(row["id"])] = df_res
dfq.at[index, "freeze_datetime_value"] = (
Expand All @@ -173,7 +173,7 @@ def update(
filename = f"{self.name}/{question_id}.jsonl"
if filename not in files_in_storage and question_id not in resolution_files:
market = self._get_market(row["id"])
df_res = self._create_resolution_file(dfq, index, market)
df_res = self._build_resolution_file(dfq, index, market)
if df_res is not None:
resolution_files[question_id] = df_res

Expand Down Expand Up @@ -325,12 +325,12 @@ def _get_resolved_market_value(market: dict) -> float:
return 0
return np.nan

def _create_resolution_file(
def _build_resolution_file(
self,
dfq: pd.DataFrame,
index: int,
market: dict,
) -> pd.DataFrame | None:
) -> DataFrame[ResolutionFrame] | None:
"""Build the resolution file for a market from its aggregation history.

Overwrites the resolution file entirely on each run (Metaculus returns the
Expand Down
64 changes: 32 additions & 32 deletions src/tests/test_metaculus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,12 @@ def test_only_keeps_id_date_value(self):


# ---------------------------------------------------------------------------
# _create_resolution_file
# _build_resolution_file
# ---------------------------------------------------------------------------


class TestCreateResolutionFile:
"""Tests for MetaculusSource._create_resolution_file."""
"""Tests for MetaculusSource._build_resolution_file."""

def _dfq_row(self, resolved=False, resolution_datetime="N/A"):
return make_question_df(
Expand All @@ -179,7 +179,7 @@ def test_empty_history_returns_none(self, metaculus_source):
question={"aggregations": {"recency_weighted": {"history": []}}}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is None

def test_single_day_entries_filtered(self, metaculus_source):
Expand All @@ -201,7 +201,7 @@ def test_single_day_entries_filtered(self, metaculus_source):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is None

def test_single_day_last_millisecond_kept(self, metaculus_source):
Expand All @@ -224,7 +224,7 @@ def test_single_day_last_millisecond_kept(self, metaculus_source):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
assert len(result) >= 1

Expand Down Expand Up @@ -252,7 +252,7 @@ def test_backfill_gaps(self, metaculus_source):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# Should have continuous dates from start through last date
dates_in_result = result["date"].tolist()
Expand Down Expand Up @@ -282,7 +282,7 @@ def test_deduplication_keeps_last(self, metaculus_source):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# Both entries map to date 2025-01-01 (end_date - 1 day)
# The second (0.9) should be kept
Expand All @@ -304,7 +304,7 @@ def test_resolved_market_truncates_and_appends(self, metaculus_source):
resolved=True,
resolution_datetime="2025-01-03T00:00:00+00:00",
)
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# Last row should be the resolution value (yes -> 1)
assert float(result.iloc[-1]["value"]) == 1.0
Expand Down Expand Up @@ -335,7 +335,7 @@ def test_null_end_time_uses_today(self, metaculus_source, freeze_today):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# Should have dates from Dec 31 through Jan 4 (today - 1 day)
assert len(result) >= 2
Expand Down Expand Up @@ -364,7 +364,7 @@ def test_future_dates_dropped(self, metaculus_source, freeze_today):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# No date may exceed yesterday (2025-01-04); the future 2025-01-09 entry is dropped.
assert max(result["date"].tolist()) <= "2025-01-04"
Expand All @@ -389,7 +389,7 @@ def test_future_only_unresolved_market_returns_none(self, metaculus_source, free
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is None

def test_date_assignment_subtracts_day(self, metaculus_source):
Expand All @@ -412,7 +412,7 @@ def test_date_assignment_subtracts_day(self, metaculus_source):
}
)
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
# end_date 2025-01-02 minus 1 day = 2025-01-01
assert "2025-01-01" in result["date"].tolist()
Expand All @@ -421,15 +421,15 @@ def test_id_is_string(self, metaculus_source):
"""Output id column contains string values."""
market = make_metaculus_market()
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
assert all(isinstance(v, str) for v in result["id"].tolist())

def test_output_validated_as_resolution_frame(self, metaculus_source):
"""Output passes ResolutionFrame.validate()."""
market = make_metaculus_market()
dfq = self._dfq_row()
result = metaculus_source._create_resolution_file(dfq, 0, market)
result = metaculus_source._build_resolution_file(dfq, 0, market)
assert result is not None
ResolutionFrame.validate(result)

Expand Down Expand Up @@ -682,7 +682,7 @@ def test_api_key_required(self):


# ---------------------------------------------------------------------------
# update() (mock _get_market and _create_resolution_file)
# update() (mock _get_market and _build_resolution_file)
# ---------------------------------------------------------------------------


Expand All @@ -699,7 +699,7 @@ def _resolution_df(self, question_id="42472", value=0.6):
}
)

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_new_question_appended(self, mock_market, mock_res, metaculus_source):
"""ID in dff not in dfq gets appended with defaults."""
Expand All @@ -715,7 +715,7 @@ def test_new_question_appended(self, mock_market, mock_res, metaculus_source):
row = result.dfq[result.dfq["id"] == "200"].iloc[0]
assert row["freeze_datetime_value_explanation"] == "The community prediction."

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_existing_question_not_duplicated(self, mock_market, mock_res, metaculus_source):
"""ID already in dfq does not add a new row."""
Expand All @@ -728,7 +728,7 @@ def test_existing_question_not_duplicated(self, mock_market, mock_res, metaculus
result = metaculus_source.update(dfq, dff)
assert len(result.dfq) == 1

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_question_fields_updated(self, mock_market, mock_res, metaculus_source):
"""Unresolved question fields are updated from market data."""
Expand All @@ -753,7 +753,7 @@ def test_question_fields_updated(self, mock_market, mock_res, metaculus_source):
assert row["market_info_resolution_criteria"] == "New criteria"
assert "metaculus.com/questions/42472" in row["url"]

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_background_empty_string_kept(self, mock_market, mock_res, metaculus_source):
"""Empty description string is stored as-is, not converted to 'N/A'."""
Expand All @@ -767,7 +767,7 @@ def test_background_empty_string_kept(self, mock_market, mock_res, metaculus_sou
result = metaculus_source.update(dfq, dff)
assert result.dfq.iloc[0]["background"] == ""

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_background_missing_key_becomes_na(self, mock_market, mock_res, metaculus_source):
"""Missing description key becomes 'N/A'."""
Expand All @@ -782,7 +782,7 @@ def test_background_missing_key_becomes_na(self, mock_market, mock_res, metaculu
result = metaculus_source.update(dfq, dff)
assert result.dfq.iloc[0]["background"] == "N/A"

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_resolved_market_sets_resolution_datetime(
self, mock_market, mock_res, metaculus_source
Expand All @@ -808,10 +808,10 @@ def test_resolved_market_sets_resolution_datetime(
# min(close=March 1, resolve=Feb 15) = Feb 15
assert "2026-02-15" in str(row["market_info_resolution_datetime"])

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_resolution_file_stored(self, mock_market, mock_res, metaculus_source):
"""Resolution file from _create_resolution_file is stored in result."""
"""Resolution file from _build_resolution_file is stored in result."""
mock_market.return_value = make_metaculus_market()
res_df = self._resolution_df()
mock_res.return_value = res_df
Expand All @@ -822,7 +822,7 @@ def test_resolution_file_stored(self, mock_market, mock_res, metaculus_source):
result = metaculus_source.update(dfq, dff)
assert "42472" in result.resolution_files

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_freeze_value_from_last_resolution(self, mock_market, mock_res, metaculus_source):
"""freeze_datetime_value is the last value in the resolution file."""
Expand All @@ -836,10 +836,10 @@ def test_freeze_value_from_last_resolution(self, mock_market, mock_res, metaculu
# QuestionFrame coerces freeze_datetime_value to str
assert str(result.dfq.iloc[0]["freeze_datetime_value"]) == "0.75"

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_freeze_value_na_when_no_resolution(self, mock_market, mock_res, metaculus_source):
"""freeze_datetime_value is 'N/A' when _create_resolution_file returns None."""
"""freeze_datetime_value is 'N/A' when _build_resolution_file returns None."""
mock_market.return_value = make_metaculus_market()
mock_res.return_value = None

Expand All @@ -849,7 +849,7 @@ def test_freeze_value_na_when_no_resolution(self, mock_market, mock_res, metacul
result = metaculus_source.update(dfq, dff)
assert result.dfq.iloc[0]["freeze_datetime_value"] == "N/A"

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_market_api_failure_propagates(self, mock_market, mock_res, metaculus_source):
"""A persistent _get_market failure propagates (fail loudly), not silently skipped."""
Expand All @@ -862,7 +862,7 @@ def test_market_api_failure_propagates(self, mock_market, mock_res, metaculus_so
metaculus_source.update(dfq, dff)
mock_res.assert_not_called()

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_resolved_missing_resolution_regenerated(self, mock_market, mock_res, metaculus_source):
"""Resolved question without existing resolution file triggers regeneration."""
Expand Down Expand Up @@ -895,7 +895,7 @@ def test_resolved_missing_resolution_regenerated(self, mock_market, mock_res, me
mock_market.assert_called_once_with("42472")
assert "42472" in result.resolution_files

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_resolved_with_existing_file_not_regenerated(
self, mock_market, mock_res, metaculus_source
Expand All @@ -917,7 +917,7 @@ def test_resolved_with_existing_file_not_regenerated(
mock_market.assert_not_called()
mock_res.assert_not_called()

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_cap_new_questions(self, mock_market, mock_res, metaculus_source):
"""New IDs exceeding _QUESTION_LIMIT - unresolved are capped."""
Expand All @@ -942,7 +942,7 @@ def test_api_key_required(self):
with pytest.raises(RuntimeError, match="api_key must be set"):
src.update(dfq, dff)

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_forecast_horizons_always_na(self, mock_market, mock_res, metaculus_source):
"""Every updated row has forecast_horizons = 'N/A'."""
Expand All @@ -955,7 +955,7 @@ def test_forecast_horizons_always_na(self, mock_market, mock_res, metaculus_sour
result = metaculus_source.update(dfq, dff)
assert result.dfq.iloc[0]["forecast_horizons"] == "N/A"

@patch.object(MetaculusSource, "_create_resolution_file")
@patch.object(MetaculusSource, "_build_resolution_file")
@patch.object(MetaculusSource, "_get_market")
def test_valid_question_frame_output(self, mock_market, mock_res, metaculus_source):
"""result.dfq passes QuestionFrame.validate()."""
Expand Down
Loading