From 4528b4421d28d3d3997d341b02fbc8d6bcce177a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Dec 2025 00:45:11 +0000 Subject: [PATCH 1/3] Refactor: Support multiple ERCOT products response shapes Co-authored-by: kvkenyon --- tests/test_ercot_http_products.py | 52 +++++++++++++++++++++++++++ tinygrid/ercot/client.py | 59 ++++++++++++++++++++++++++++--- 2 files changed, 106 insertions(+), 5 deletions(-) diff --git a/tests/test_ercot_http_products.py b/tests/test_ercot_http_products.py index 5cb1d98..bb3a2fb 100644 --- a/tests/test_ercot_http_products.py +++ b/tests/test_ercot_http_products.py @@ -41,6 +41,58 @@ def test_get_list_for_products_dispatches_correct_url(self): request = route.calls.last.request assert request.url.path == "/api/public-reports/" + +class TestProductsResponseShapes: + """Regression tests for products response parsing.""" + + def test_products_to_dataframe_supports_hal_embedded_shape(self): + """HAL responses store the list under _embedded.products.""" + ercot = ERCOT() + response = { + "_embedded": { + "products": [ + { + "emilId": "np6-905-cd", + "name": "SPP Node Zone Hub", + "description": "Settlement Point Prices", + } + ] + }, + "_links": {"self": {"href": "/api/public-reports/"}}, + } + df = ercot._products_to_dataframe(response) + assert not df.empty + assert df.loc[0, "emilId"] == "np6-905-cd" + + def test_products_to_dataframe_supports_nested_additional_properties_embedded_shape( + self, + ): + """Some pyercot model to_dict() outputs keep HAL under additional_properties.""" + ercot = ERCOT() + response = { + "additional_properties": { + "_embedded": { + "products": [ + { + "emilId": "np6-905-cd", + "name": "SPP Node Zone Hub", + } + ] + } + } + } + df = ercot._products_to_dataframe(response) + assert not df.empty + assert df.loc[0, "emilId"] == "np6-905-cd" + + def test_products_to_dataframe_supports_raw_list_shape(self): + """Some clients can return a raw list of product dicts.""" + ercot = ERCOT() + response = [{"emilId": "np6-905-cd", "name": "SPP Node Zone Hub"}] + df = ercot._products_to_dataframe(response) + assert not df.empty + assert df.loc[0, "emilId"] == "np6-905-cd" + @respx.mock def test_get_product_dispatches_correct_url_with_emil_id(self): """Test get_product calls the correct endpoint with emil_id in path.""" diff --git a/tinygrid/ercot/client.py b/tinygrid/ercot/client.py index 72c2220..2e58b63 100644 --- a/tinygrid/ercot/client.py +++ b/tinygrid/ercot/client.py @@ -644,12 +644,61 @@ def _call_endpoint_model( """ return self._call_with_retry(endpoint_module, endpoint_name, **kwargs) - def _products_to_dataframe(self, response: dict[str, Any]) -> pd.DataFrame: - """Convert products list response to DataFrame.""" - products = response.get("products", []) - if not products: + def _products_to_dataframe(self, response: Any) -> pd.DataFrame: + """Convert products list response to DataFrame. + + The ERCOT products endpoint can return multiple shapes depending on the + upstream client (pyercot) and API format: + - Plain dict: {"products": [...]} + - HAL dict: {"_embedded": {"products": [...]}, ...} + - Nested HAL in to_dict(): {"additional_properties": {"_embedded": {"products": [...]}}} + - Raw list: [...] + """ + + def _as_products_list(value: Any) -> list[dict[str, Any]]: + if not value: + return [] + if isinstance(value, list): + # Best effort: only keep mapping-like entries + return [item for item in value if isinstance(item, dict)] + return [] + + if response is None: return pd.DataFrame() - return pd.DataFrame(products) + + # Some clients can return a raw list response + if isinstance(response, list): + products = _as_products_list(response) + return pd.DataFrame(products) if products else pd.DataFrame() + + if not isinstance(response, dict): + return pd.DataFrame() + + # Common shape: {"products": [...]} + products = _as_products_list(response.get("products")) + if products: + return pd.DataFrame(products) + + # HAL shape: {"_embedded": {"products": [...]}} + embedded = response.get("_embedded") + if isinstance(embedded, dict): + products = _as_products_list(embedded.get("products")) + if products: + return pd.DataFrame(products) + + # Some model to_dict() outputs store HAL payload under additional_properties + additional_properties = response.get("additional_properties") + if isinstance(additional_properties, dict): + products = _as_products_list(additional_properties.get("products")) + if products: + return pd.DataFrame(products) + embedded = additional_properties.get("_embedded") + if isinstance(embedded, dict): + products = _as_products_list(embedded.get("products")) + if products: + return pd.DataFrame(products) + + return pd.DataFrame() def _model_to_dataframe(self, response: dict[str, Any]) -> pd.DataFrame: """Convert a single model response to a one-row DataFrame.""" From 04aad12729f5523aca7b389b97d082c523b208ea Mon Sep 17 00:00:00 2001 From: Kevin Kenyon Date: Sun, 28 Dec 2025 20:18:54 -0600 Subject: [PATCH 2/3] feat: Add ERCOT data access gap implementations Implements features identified in ERCOT data access gap analysis: Dashboard endpoints (dashboard.py): - Implement actual HTTP calls to ERCOT dashboard JSON endpoints - Add get_status(), get_fuel_mix(), get_renewable_generation() - Add get_supply_demand(), get_daily_prices(), get_capacity_committed() - New dataclasses: GridStatus, FuelMixEntry, RenewableStatus Rate limiting (rate_limiter.py): - Add token bucket RateLimiter and AsyncRateLimiter classes - Enforce ERCOT's 30 req/min limit proactively - Integrate into ERCOTBase._call_endpoint_raw() 5-minute resolution (api.py): - Add resolution parameter to get_wind_forecast() and get_solar_forecast() - Support "hourly" (default) and "5min" resolutions - Route to appropriate NP4-733/738/743/746 endpoints New endpoints (api.py): - Add get_dc_tie_flows() for DC tie data (NP6-626-CD) - Add get_total_generation() for system generation (NP6-625-CD) - Add get_system_wide_actuals() for SCED actuals (NP6-235-CD) EIA integration (eia.py): - Add EIAClient for supplementary ERCOT data via EIA API - Methods: get_demand(), get_generation(), get_generation_by_fuel() - Useful for data before December 2023 Polling utilities (polling.py): - Add ERCOTPoller class for continuous data monitoring - Support callback and generator patterns - Include exponential backoff and rate limit awareness Documentation: - Add API data availability constants and developer resource URLs - Update CLAUDE.md with data availability limitations - Update README.md and tinygrid/README.md with new features Tests: - Add comprehensive tests for all new modules - Coverage increased from 79% to 95% - 746 total tests (150 new tests added) --- CLAUDE.md | 33 ++ README.md | 122 +++++- tests/test_eia.py | 368 ++++++++++++++++ tests/test_ercot_api_methods.py | 268 ++++++++++++ tests/test_ercot_dashboard.py | 526 +++++++++++++++++++++-- tests/test_polling.py | 414 ++++++++++++++++++ tests/test_rate_limiter.py | 316 ++++++++++++++ tinygrid/README.md | 69 ++- tinygrid/constants/ercot.py | 61 ++- tinygrid/ercot/__init__.py | 25 +- tinygrid/ercot/api.py | 326 +++++++++++++-- tinygrid/ercot/client.py | 28 ++ tinygrid/ercot/dashboard.py | 719 +++++++++++++++++++++++++++----- tinygrid/ercot/eia.py | 368 ++++++++++++++++ tinygrid/ercot/polling.py | 343 +++++++++++++++ tinygrid/utils/__init__.py | 14 + tinygrid/utils/rate_limiter.py | 285 +++++++++++++ 17 files changed, 4099 insertions(+), 186 deletions(-) create mode 100644 tests/test_eia.py create mode 100644 tests/test_ercot_api_methods.py create mode 100644 tests/test_polling.py create mode 100644 tests/test_rate_limiter.py create mode 100644 tinygrid/ercot/eia.py create mode 100644 tinygrid/ercot/polling.py create mode 100644 tinygrid/utils/rate_limiter.py diff --git a/CLAUDE.md b/CLAUDE.md index ab8e1f8..306bc29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,35 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Tiny Grid is a Python SDK for accessing electricity grid data from US Independent System Operators (ISOs). Currently supports 100+ ERCOT endpoints with plans to support other ISOs (CAISO, PJM, NYISO, ISO-NE, MISO, SPP). +## ERCOT Developer Resources + +- **Developer Portal**: https://developer.ercot.com +- **API Explorer**: https://apiexplorer.ercot.com (register for subscription key) +- **OpenAPI Specs**: https://github.com/ercot/api-specs +- **API Base URL**: https://api.ercot.com/api/public-reports + +## ERCOT API Data Availability (Important Limitations) + +1. **API Data Start Date**: December 11, 2023 + - The REST API only contains data from this date forward + - For earlier data, use MIS document downloads or historical archives + - Historical archives extend 7+ years back + +2. **Data Delay**: Approximately 1 hour + - Real-time data is NOT truly real-time + - There is ~1 hour delay from actual grid operations to API availability + +3. **Geographic Restriction**: US IP addresses only + - API blocks requests from non-US IP addresses + - Users outside the US need a VPN or US-based proxy + +4. **Rate Limit**: 30 requests per minute + - Exceeding this limit results in HTTP 429 errors + - SDK includes built-in rate limiter (enabled by default) + +5. **Bulk Download Limit**: 1,000 documents per request + - Archive bulk downloads limited to 1,000 files per POST request + ## Architecture The project follows a three-layer architecture: @@ -95,6 +124,7 @@ uv run pyright The `ERCOT` class is the main entry point. It wraps pyercot's generated client with: - Automatic token management via `ERCOTAuth` - Retry logic with exponential backoff (configurable via `max_retries`) +- Rate limiting (30 requests/minute, configurable via `rate_limit_enabled` and `requests_per_minute`) - Context manager support for resource cleanup - Pagination handling (page_size defaults to 10000) @@ -108,6 +138,9 @@ auth = ERCOTAuth(ERCOTAuthConfig( subscription_key="key" )) ercot = ERCOT(auth=auth, max_retries=3) + +# Disable rate limiting for testing (not recommended for production) +ercot = ERCOT(auth=auth, rate_limit_enabled=False) ``` ### Data Fetching Patterns diff --git a/README.md b/README.md index ad28291..aba6a67 100644 --- a/README.md +++ b/README.md @@ -66,11 +66,13 @@ ercot = ERCOT() # Get actual system load by weather zone load = ercot.get_load(start="today", by="weather_zone") -# Get wind generation forecast +# Get wind generation forecast (hourly or 5-minute resolution) wind = ercot.get_wind_forecast(start="today") +wind_5min = ercot.get_wind_forecast(start="today", resolution="5min") # Get solar generation forecast solar = ercot.get_solar_forecast(start="today") +solar_5min = ercot.get_solar_forecast(start="today", resolution="5min", by_region=True) # Get load forecast load_forecast = ercot.get_load_forecast_by_weather_zone( @@ -79,6 +81,33 @@ load_forecast = ercot.get_load_forecast_by_weather_zone( ) ``` +### Dashboard Data (No Authentication Required) + +Access real-time grid status from ERCOT's public dashboard: + +```python +from tinygrid import ERCOT + +ercot = ERCOT() + +# Get current grid status +status = ercot.get_status() +print(f"Condition: {status.condition}") +print(f"Load: {status.current_load:,.0f} MW") +print(f"Reserves: {status.reserves:,.0f} MW") + +# Get current fuel mix +fuel_mix = ercot.get_fuel_mix() + +# Get renewable generation status +renewables = ercot.get_renewable_generation() +print(f"Wind: {renewables.wind_mw:,.0f} MW") +print(f"Solar: {renewables.solar_mw:,.0f} MW") + +# Get supply/demand data +supply_demand = ercot.get_supply_demand() +``` + ### Historical Yearly Data Access complete yearly historical data from ERCOT's MIS document system: @@ -128,6 +157,50 @@ forecast = ercot.get_load_forecast_by_weather_zone( ) ``` +### Polling for Real-Time Updates + +For continuous data monitoring, use the polling utilities: + +```python +from tinygrid import ERCOT +from tinygrid.ercot import ERCOTPoller, poll_latest + +ercot = ERCOT(auth=auth) + +# Simple generator pattern +for df in poll_latest(ercot, ercot.get_spp, interval=60, max_iterations=10): + print(f"Latest prices: {len(df)} rows") + +# Using ERCOTPoller with callback +poller = ERCOTPoller(client=ercot, interval=60) + +def handle_data(result): + if result.success: + print(f"Got {len(result.data)} rows at {result.timestamp}") + +poller.poll(method=ercot.get_spp, callback=handle_data, max_iterations=10) +``` + +### EIA Integration (Supplementary Data) + +Access ERCOT data via the EIA API for historical data before December 2023: + +```python +from tinygrid.ercot import EIAClient + +# Requires free API key from https://www.eia.gov/opendata/register.php +eia = EIAClient(api_key="your-eia-key") + +# Get hourly demand +demand = eia.get_demand(start="2022-01-01", end="2022-01-07") + +# Get generation by fuel type +gen_by_fuel = eia.get_generation_by_fuel(start="2022-01-01") + +# Get net interchange +interchange = eia.get_interchange(start="2022-01-01") +``` + See [`examples/ercot_demo.ipynb`](examples/ercot_demo.ipynb) for complete examples. ## Unified API Methods @@ -148,10 +221,28 @@ These methods provide a simpler interface with automatic routing, date parsing, | Method | Description | |--------|-------------| -| `get_wind_forecast()` | Wind power forecast (system-wide or by region) | -| `get_solar_forecast()` | Solar power forecast (system-wide or by region) | +| `get_wind_forecast()` | Wind power forecast (hourly or 5-minute, system-wide or by region) | +| `get_solar_forecast()` | Solar power forecast (hourly or 5-minute, system-wide or by region) | | `get_load()` | Actual system load by weather or forecast zone | +### System-Wide Data Methods + +| Method | Description | +|--------|-------------| +| `get_dc_tie_flows()` | DC tie flow data (connections to Eastern Interconnection/Mexico) | +| `get_total_generation()` | Total ERCOT system generation | +| `get_system_wide_actuals()` | System-wide actual values per SCED interval | + +### Dashboard Methods (No Auth Required) + +| Method | Description | +|--------|-------------| +| `get_status()` | Grid operating condition, load, capacity, reserves | +| `get_fuel_mix()` | Current generation by fuel type | +| `get_renewable_generation()` | Wind and solar output with forecasts | +| `get_supply_demand()` | Hourly supply/demand data | +| `get_daily_prices()` | Daily price summary | + ### Direct Endpoint Methods For full control, 100+ low-level endpoint methods are available: @@ -177,6 +268,9 @@ For full control, 100+ low-level endpoint methods are available: - **Location filtering**: Filter by load zones, trading hubs, or specific settlement points - **Market selection**: Choose between real-time and day-ahead markets - **Standardized columns**: Consistent column names across all endpoints +- **Rate limiting**: Built-in rate limiter (30 req/min) to prevent API throttling +- **5-minute resolution**: Wind and solar forecasts available in 5-minute granularity +- **Retry with backoff**: Automatic retry for transient failures ## ERCOT API Credentials @@ -188,6 +282,20 @@ Authentication is required for some endpoints. To get credentials: **Note:** Dashboard methods (`get_status()`, `get_fuel_mix()`, etc.) do not require authentication. +## API Data Availability + +Important limitations to be aware of: + +| Limitation | Details | +|------------|---------| +| **API Data Start Date** | December 11, 2023 - use archive API or EIA for earlier data | +| **Data Delay** | ~1 hour from real-time to API availability | +| **Geographic Restriction** | US IP addresses only (VPN required for international) | +| **Rate Limit** | 30 requests per minute (built-in rate limiter enforces this) | +| **Bulk Download Limit** | 1,000 documents per archive request | + +For data before December 2023, use `get_rtm_spp_historical()`, `get_dam_spp_historical()`, or the EIA integration. + ## Available ERCOT Endpoints Direct access to 100+ ERCOT endpoints organized by category: @@ -247,20 +355,22 @@ tinygrid/ ├── tinygrid/ # SDK layer │ ├── ercot/ # ERCOT client package │ │ ├── __init__.py # Main ERCOT class (combining mixins) -│ │ ├── client.py # ERCOTBase with auth, retry, pagination +│ │ ├── client.py # ERCOTBase with auth, retry, pagination, rate limiting │ │ ├── endpoints.py # Low-level pyercot wrappers (~100 methods) │ │ ├── api.py # High-level unified API methods │ │ ├── archive.py # Historical archive access │ │ ├── dashboard.py # Public dashboard methods (no auth) │ │ ├── documents.py # MIS document fetching +│ │ ├── eia.py # EIA API integration for supplementary data +│ │ ├── polling.py # Real-time polling utilities │ │ └── transforms.py # Data filtering/transformation utilities │ ├── auth/ # Authentication handling │ ├── constants/ # Market types, location enums, endpoint mappings -│ ├── utils/ # Date parsing, timezone handling, decorators +│ ├── utils/ # Date parsing, timezone, decorators, rate limiting │ └── errors.py # Error types ├── pyercot/ # Auto-generated ERCOT API client (from OpenAPI spec) ├── examples/ # Usage examples -└── tests/ # Test suite (505 tests) +└── tests/ # Test suite (746 tests, 95% coverage) ``` ## Development diff --git a/tests/test_eia.py b/tests/test_eia.py new file mode 100644 index 0000000..c7cdd15 --- /dev/null +++ b/tests/test_eia.py @@ -0,0 +1,368 @@ +"""Tests for tinygrid.ercot.eia module.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from tinygrid.ercot.eia import ( + EIA_API_BASE_URL, + EIA_BULK_DOWNLOAD_URL, + ERCOT_BA_CODE, + EIAClient, + _map_fuel_type, +) + + +class TestEIAConstants: + """Tests for EIA constants.""" + + def test_eia_api_base_url(self): + """Test EIA API base URL.""" + assert EIA_API_BASE_URL == "https://api.eia.gov/v2" + + def test_ercot_ba_code(self): + """Test ERCOT balancing authority code.""" + assert ERCOT_BA_CODE == "ERCO" + + def test_eia_bulk_download_url(self): + """Test EIA bulk download URL.""" + assert EIA_BULK_DOWNLOAD_URL == "https://www.eia.gov/opendata/bulk/EBA.zip" + + +class TestMapFuelType: + """Tests for _map_fuel_type function.""" + + def test_map_coal(self): + """Test mapping coal fuel type.""" + assert _map_fuel_type("COL") == "coal" + assert _map_fuel_type("col") == "coal" + + def test_map_natural_gas(self): + """Test mapping natural gas fuel type.""" + assert _map_fuel_type("NG") == "natural_gas" + assert _map_fuel_type("ng") == "natural_gas" + + def test_map_nuclear(self): + """Test mapping nuclear fuel type.""" + assert _map_fuel_type("NUC") == "nuclear" + + def test_map_oil(self): + """Test mapping oil fuel type.""" + assert _map_fuel_type("OIL") == "oil" + + def test_map_hydro(self): + """Test mapping hydro fuel type.""" + assert _map_fuel_type("WAT") == "hydro" + + def test_map_wind(self): + """Test mapping wind fuel type.""" + assert _map_fuel_type("WND") == "wind" + + def test_map_solar(self): + """Test mapping solar fuel type.""" + assert _map_fuel_type("SUN") == "solar" + + def test_map_other(self): + """Test mapping other fuel type.""" + assert _map_fuel_type("OTH") == "other" + + def test_map_unknown(self): + """Test mapping unknown fuel type.""" + assert _map_fuel_type("UNK") == "unknown" + + def test_map_unmapped_returns_lowercase(self): + """Test unmapped fuel types return lowercase.""" + assert _map_fuel_type("XYZ") == "xyz" + assert _map_fuel_type("NewType") == "newtype" + + +class TestEIAClientInit: + """Tests for EIAClient initialization.""" + + def test_init_with_api_key(self): + """Test initialization with API key.""" + client = EIAClient(api_key="test-key") + assert client.api_key == "test-key" + assert client.timeout == 30.0 + + def test_init_with_custom_timeout(self): + """Test initialization with custom timeout.""" + client = EIAClient(api_key="test-key", timeout=60.0) + assert client.timeout == 60.0 + + def test_init_without_api_key(self): + """Test initialization without API key.""" + client = EIAClient() + assert client.api_key is None + + +class TestEIAClientMakeRequest: + """Tests for EIAClient._make_request method.""" + + def test_make_request_without_api_key_raises(self): + """Test that make_request raises without API key.""" + client = EIAClient() + + with pytest.raises(ValueError, match="EIA API key required"): + client._make_request("test-endpoint") + + @patch("tinygrid.ercot.eia.httpx.Client") + def test_make_request_success(self, mock_client_class): + """Test successful API request.""" + mock_response = MagicMock() + mock_response.json.return_value = {"response": {"data": []}} + mock_client = MagicMock() + mock_client.get.return_value = mock_response + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + client = EIAClient(api_key="test-key") + result = client._make_request("test-endpoint", {"param": "value"}) + + assert result == {"response": {"data": []}} + mock_client.get.assert_called_once() + + @patch("tinygrid.ercot.eia.httpx.Client") + def test_make_request_timeout(self, mock_client_class): + """Test request timeout handling.""" + import httpx + + mock_client = MagicMock() + mock_client.get.side_effect = httpx.TimeoutException("Timeout") + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + client = EIAClient(api_key="test-key") + + with pytest.raises(httpx.TimeoutException): + client._make_request("test-endpoint") + + @patch("tinygrid.ercot.eia.httpx.Client") + def test_make_request_http_error(self, mock_client_class): + """Test HTTP error handling.""" + import httpx + + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Server Error", + request=MagicMock(), + response=mock_response, + ) + mock_client = MagicMock() + mock_client.get.return_value = mock_response + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + client = EIAClient(api_key="test-key") + + with pytest.raises(httpx.HTTPStatusError): + client._make_request("test-endpoint") + + +class TestEIAClientGetDemand: + """Tests for EIAClient.get_demand method.""" + + @patch.object(EIAClient, "_make_request") + def test_get_demand_success(self, mock_request): + """Test successful demand data fetch.""" + mock_request.return_value = { + "response": { + "data": [ + {"period": "2024-01-01T12:00:00", "value": 50000}, + {"period": "2024-01-01T13:00:00", "value": 51000}, + ] + } + } + + client = EIAClient(api_key="test-key") + result = client.get_demand(start="2024-01-01", end="2024-01-02") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 2 + assert "timestamp" in result.columns + assert "demand_mw" in result.columns + assert result["demand_mw"].iloc[0] == 50000 + + @patch.object(EIAClient, "_make_request") + def test_get_demand_empty_response(self, mock_request): + """Test empty demand response.""" + mock_request.return_value = {"response": {"data": []}} + + client = EIAClient(api_key="test-key") + result = client.get_demand(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + @patch.object(EIAClient, "_make_request") + def test_get_demand_with_default_end(self, mock_request): + """Test demand fetch with default end date.""" + mock_request.return_value = {"response": {"data": []}} + + client = EIAClient(api_key="test-key") + result = client.get_demand(start="2024-01-01") + + # Should have called with end = start + 7 days + assert isinstance(result, pd.DataFrame) + + @patch.object(EIAClient, "_make_request") + def test_get_demand_exception_returns_empty(self, mock_request): + """Test that exceptions return empty DataFrame.""" + mock_request.side_effect = Exception("API Error") + + client = EIAClient(api_key="test-key") + result = client.get_demand(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + assert "demand_mw" in result.columns + + +class TestEIAClientGetGeneration: + """Tests for EIAClient.get_generation method.""" + + @patch.object(EIAClient, "_make_request") + def test_get_generation_success(self, mock_request): + """Test successful generation data fetch.""" + mock_request.return_value = { + "response": { + "data": [ + {"period": "2024-01-01T12:00:00", "value": 48000}, + {"period": "2024-01-01T13:00:00", "value": 49000}, + ] + } + } + + client = EIAClient(api_key="test-key") + result = client.get_generation(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 2 + assert "generation_mw" in result.columns + + @patch.object(EIAClient, "_make_request") + def test_get_generation_empty_response(self, mock_request): + """Test empty generation response.""" + mock_request.return_value = {"response": {"data": []}} + + client = EIAClient(api_key="test-key") + result = client.get_generation(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + @patch.object(EIAClient, "_make_request") + def test_get_generation_exception_returns_empty(self, mock_request): + """Test that exceptions return empty DataFrame.""" + mock_request.side_effect = Exception("API Error") + + client = EIAClient(api_key="test-key") + result = client.get_generation(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + +class TestEIAClientGetGenerationByFuel: + """Tests for EIAClient.get_generation_by_fuel method.""" + + @patch.object(EIAClient, "_make_request") + def test_get_generation_by_fuel_success(self, mock_request): + """Test successful fuel mix data fetch.""" + mock_request.return_value = { + "response": { + "data": [ + {"period": "2024-01-01T12:00:00", "fueltype": "NG", "value": 25000}, + { + "period": "2024-01-01T12:00:00", + "fueltype": "WND", + "value": 18000, + }, + ] + } + } + + client = EIAClient(api_key="test-key") + result = client.get_generation_by_fuel(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 2 + assert "fuel_type" in result.columns + assert "generation_mw" in result.columns + assert result["fuel_type"].iloc[0] == "natural_gas" + assert result["fuel_type"].iloc[1] == "wind" + + @patch.object(EIAClient, "_make_request") + def test_get_generation_by_fuel_empty(self, mock_request): + """Test empty fuel mix response.""" + mock_request.return_value = {"response": {"data": []}} + + client = EIAClient(api_key="test-key") + result = client.get_generation_by_fuel(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + @patch.object(EIAClient, "_make_request") + def test_get_generation_by_fuel_exception(self, mock_request): + """Test that exceptions return empty DataFrame.""" + mock_request.side_effect = Exception("API Error") + + client = EIAClient(api_key="test-key") + result = client.get_generation_by_fuel(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + +class TestEIAClientGetInterchange: + """Tests for EIAClient.get_interchange method.""" + + @patch.object(EIAClient, "_make_request") + def test_get_interchange_success(self, mock_request): + """Test successful interchange data fetch.""" + mock_request.return_value = { + "response": { + "data": [ + {"period": "2024-01-01T12:00:00", "value": 500}, + {"period": "2024-01-01T13:00:00", "value": -200}, + ] + } + } + + client = EIAClient(api_key="test-key") + result = client.get_interchange(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 2 + assert "interchange_mw" in result.columns + assert result["interchange_mw"].iloc[0] == 500 + + @patch.object(EIAClient, "_make_request") + def test_get_interchange_empty(self, mock_request): + """Test empty interchange response.""" + mock_request.return_value = {"response": {"data": []}} + + client = EIAClient(api_key="test-key") + result = client.get_interchange(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 + + @patch.object(EIAClient, "_make_request") + def test_get_interchange_exception(self, mock_request): + """Test that exceptions return empty DataFrame.""" + mock_request.side_effect = Exception("API Error") + + client = EIAClient(api_key="test-key") + result = client.get_interchange(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + assert len(result) == 0 diff --git a/tests/test_ercot_api_methods.py b/tests/test_ercot_api_methods.py new file mode 100644 index 0000000..7535527 --- /dev/null +++ b/tests/test_ercot_api_methods.py @@ -0,0 +1,268 @@ +"""Tests for new ERCOT API methods (dc_tie, total_gen, system_actuals, 5min resolution).""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from tinygrid.ercot.api import ERCOTAPIMixin + + +class TestableERCOTAPIMixin(ERCOTAPIMixin): + """Testable version of ERCOTAPIMixin with mocked dependencies.""" + + def __init__(self): + self._mock_archive = MagicMock() + self._mock_archive.fetch_historical = MagicMock(return_value=pd.DataFrame()) + + def _get_archive(self): + return self._mock_archive + + def _needs_historical(self, date, data_type): + return True # Always use archive for testing + + # Mock the pyercot endpoint methods + def get_wpp_hourly_average_actual_forecast(self, **kwargs): + return pd.DataFrame() + + def get_wpp_hourly_actual_forecast_geo(self, **kwargs): + return pd.DataFrame() + + def get_wpp_actual_5min_avg_values(self, **kwargs): + return pd.DataFrame() + + def get_wpp_actual_5min_avg_values_geo(self, **kwargs): + return pd.DataFrame() + + def get_spp_hourly_average_actual_forecast(self, **kwargs): + return pd.DataFrame() + + def get_spp_hourly_actual_forecast_geo(self, **kwargs): + return pd.DataFrame() + + def get_spp_actual_5min_avg_values(self, **kwargs): + return pd.DataFrame() + + def get_spp_actual_5min_avg_values_geo(self, **kwargs): + return pd.DataFrame() + + +class TestGetDCTieFlows: + """Tests for get_dc_tie_flows method.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_get_dc_tie_flows_returns_dataframe(self, mixin): + """Test get_dc_tie_flows returns DataFrame.""" + mixin._mock_archive.fetch_historical.return_value = pd.DataFrame( + {"col1": [1, 2], "col2": [3, 4]} + ) + + result = mixin.get_dc_tie_flows(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + + def test_get_dc_tie_flows_calls_archive(self, mixin): + """Test get_dc_tie_flows uses archive API.""" + mixin.get_dc_tie_flows(start="2024-01-01") + + mixin._mock_archive.fetch_historical.assert_called_once() + call_args = mixin._mock_archive.fetch_historical.call_args + assert "/np6-626-cd/dc_tie" in call_args[1]["endpoint"] + + def test_get_dc_tie_flows_with_date_range(self, mixin): + """Test get_dc_tie_flows with start and end dates.""" + mixin.get_dc_tie_flows(start="2024-01-01", end="2024-01-07") + + call_args = mixin._mock_archive.fetch_historical.call_args + assert call_args[1]["start"] is not None + assert call_args[1]["end"] is not None + + +class TestGetTotalGeneration: + """Tests for get_total_generation method.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_get_total_generation_returns_dataframe(self, mixin): + """Test get_total_generation returns DataFrame.""" + mixin._mock_archive.fetch_historical.return_value = pd.DataFrame( + {"col1": [1, 2], "col2": [3, 4]} + ) + + result = mixin.get_total_generation(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + + def test_get_total_generation_calls_archive(self, mixin): + """Test get_total_generation uses archive API.""" + mixin.get_total_generation(start="2024-01-01") + + mixin._mock_archive.fetch_historical.assert_called_once() + call_args = mixin._mock_archive.fetch_historical.call_args + assert "/np6-625-cd/se_totalgen" in call_args[1]["endpoint"] + + +class TestGetSystemWideActuals: + """Tests for get_system_wide_actuals method.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_get_system_wide_actuals_returns_dataframe(self, mixin): + """Test get_system_wide_actuals returns DataFrame.""" + mixin._mock_archive.fetch_historical.return_value = pd.DataFrame( + {"col1": [1, 2], "col2": [3, 4]} + ) + + result = mixin.get_system_wide_actuals(start="2024-01-01") + + assert isinstance(result, pd.DataFrame) + + def test_get_system_wide_actuals_calls_archive(self, mixin): + """Test get_system_wide_actuals uses archive API.""" + mixin.get_system_wide_actuals(start="2024-01-01") + + mixin._mock_archive.fetch_historical.assert_called_once() + call_args = mixin._mock_archive.fetch_historical.call_args + assert "/np6-235-cd/sys_wide_actuals" in call_args[1]["endpoint"] + + +class TestWindForecastResolution: + """Tests for get_wind_forecast 5-minute resolution.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_wind_forecast_invalid_resolution_raises(self, mixin): + """Test that invalid resolution raises ValueError.""" + with pytest.raises(ValueError, match="Invalid resolution"): + mixin.get_wind_forecast(start="2024-01-01", resolution="invalid") + + def test_wind_forecast_5min_resolution(self, mixin): + """Test 5-minute resolution uses correct endpoint.""" + mixin.get_wind_forecast(start="2024-01-01", resolution="5min") + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-733-cd" in call_args[1]["endpoint"] + + def test_wind_forecast_5min_by_region(self, mixin): + """Test 5-minute resolution by region uses correct endpoint.""" + mixin.get_wind_forecast(start="2024-01-01", resolution="5min", by_region=True) + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-743-cd" in call_args[1]["endpoint"] + + def test_wind_forecast_hourly_resolution(self, mixin): + """Test hourly resolution uses correct endpoint.""" + mixin.get_wind_forecast(start="2024-01-01", resolution="hourly") + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-732-cd" in call_args[1]["endpoint"] + + def test_wind_forecast_accepts_various_5min_formats(self, mixin): + """Test 5-minute resolution accepts various formats.""" + # Test different formats + for fmt in ["5min", "5-min", "5_min"]: + mixin._mock_archive.fetch_historical.reset_mock() + mixin.get_wind_forecast(start="2024-01-01", resolution=fmt) + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-733-cd" in call_args[1]["endpoint"] + + +class TestSolarForecastResolution: + """Tests for get_solar_forecast 5-minute resolution.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_solar_forecast_invalid_resolution_raises(self, mixin): + """Test that invalid resolution raises ValueError.""" + with pytest.raises(ValueError, match="Invalid resolution"): + mixin.get_solar_forecast(start="2024-01-01", resolution="invalid") + + def test_solar_forecast_5min_resolution(self, mixin): + """Test 5-minute resolution uses correct endpoint.""" + mixin.get_solar_forecast(start="2024-01-01", resolution="5min") + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-738-cd" in call_args[1]["endpoint"] + + def test_solar_forecast_5min_by_region(self, mixin): + """Test 5-minute resolution by region uses correct endpoint.""" + mixin.get_solar_forecast(start="2024-01-01", resolution="5min", by_region=True) + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-746-cd" in call_args[1]["endpoint"] + + def test_solar_forecast_hourly_resolution(self, mixin): + """Test hourly resolution uses correct endpoint.""" + mixin.get_solar_forecast(start="2024-01-01", resolution="hourly") + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-737-cd" in call_args[1]["endpoint"] + + +class TestAPIMethodDefaults: + """Tests for API method default behaviors.""" + + @pytest.fixture + def mixin(self): + return TestableERCOTAPIMixin() + + def test_wind_forecast_default_resolution_is_hourly(self, mixin): + """Test wind forecast defaults to hourly resolution.""" + # Call without specifying resolution + mixin.get_wind_forecast(start="2024-01-01") + + call_args = mixin._mock_archive.fetch_historical.call_args + # Should use hourly endpoint + assert "np4-732-cd" in call_args[1]["endpoint"] + + def test_solar_forecast_default_resolution_is_hourly(self, mixin): + """Test solar forecast defaults to hourly resolution.""" + # Call without specifying resolution + mixin.get_solar_forecast(start="2024-01-01") + + call_args = mixin._mock_archive.fetch_historical.call_args + # Should use hourly endpoint + assert "np4-737-cd" in call_args[1]["endpoint"] + + def test_wind_forecast_default_by_region_is_false(self, mixin): + """Test wind forecast defaults to non-regional data.""" + mixin.get_wind_forecast(start="2024-01-01") + + call_args = mixin._mock_archive.fetch_historical.call_args + # Should NOT use geo endpoint + assert "geo" not in call_args[1]["endpoint"] + + def test_solar_forecast_default_by_region_is_false(self, mixin): + """Test solar forecast defaults to non-regional data.""" + mixin.get_solar_forecast(start="2024-01-01") + + call_args = mixin._mock_archive.fetch_historical.call_args + # Should NOT use geo endpoint + assert "geo" not in call_args[1]["endpoint"] + + def test_wind_forecast_hourly_by_region(self, mixin): + """Test hourly wind forecast by region.""" + mixin.get_wind_forecast(start="2024-01-01", by_region=True) + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-742-cd" in call_args[1]["endpoint"] + + def test_solar_forecast_hourly_by_region(self, mixin): + """Test hourly solar forecast by region.""" + mixin.get_solar_forecast(start="2024-01-01", by_region=True) + + call_args = mixin._mock_archive.fetch_historical.call_args + assert "np4-745-cd" in call_args[1]["endpoint"] diff --git a/tests/test_ercot_dashboard.py b/tests/test_ercot_dashboard.py index caad481..b8cf1c0 100644 --- a/tests/test_ercot_dashboard.py +++ b/tests/test_ercot_dashboard.py @@ -2,13 +2,20 @@ from __future__ import annotations +from unittest.mock import patch + import pandas as pd import pytest from tinygrid.ercot.dashboard import ( ERCOTDashboardMixin, + FuelMixEntry, GridCondition, GridStatus, + RenewableStatus, + _fetch_json, + _parse_timestamp, + _safe_float, ) @@ -88,51 +95,514 @@ class TestClass(ERCOTDashboardMixin): return TestClass() - def test_get_status_returns_unavailable(self, mixin_instance): - """Test get_status returns unavailable GridStatus.""" + def test_get_status_returns_grid_status(self, mixin_instance): + """Test get_status returns GridStatus object.""" status = mixin_instance.get_status() assert isinstance(status, GridStatus) + # Status may be UNKNOWN if API is unavailable, or a real condition if it works + assert isinstance(status.condition, GridCondition) + + def test_get_fuel_mix_returns_dataframe(self, mixin_instance): + """Test get_fuel_mix returns DataFrame (may be empty if API unavailable).""" + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + + def test_get_fuel_mix_with_as_dataframe_false(self, mixin_instance): + """Test get_fuel_mix returns list when as_dataframe=False.""" + result = mixin_instance.get_fuel_mix(as_dataframe=False) + assert isinstance(result, list) + + def test_get_energy_storage_resources_returns_dataframe(self, mixin_instance): + """Test get_energy_storage_resources returns DataFrame.""" + df = mixin_instance.get_energy_storage_resources() + assert isinstance(df, pd.DataFrame) + + def test_get_system_wide_demand_returns_dataframe(self, mixin_instance): + """Test get_system_wide_demand returns DataFrame.""" + df = mixin_instance.get_system_wide_demand() + assert isinstance(df, pd.DataFrame) + + def test_get_renewable_generation_returns_status(self, mixin_instance): + """Test get_renewable_generation returns RenewableStatus.""" + result = mixin_instance.get_renewable_generation() + assert isinstance(result, RenewableStatus) + assert hasattr(result, "wind_mw") + assert hasattr(result, "solar_mw") + + def test_get_capacity_committed_returns_dataframe(self, mixin_instance): + """Test get_capacity_committed returns DataFrame.""" + df = mixin_instance.get_capacity_committed() + assert isinstance(df, pd.DataFrame) + + def test_get_capacity_forecast_returns_dataframe(self, mixin_instance): + """Test get_capacity_forecast returns DataFrame.""" + df = mixin_instance.get_capacity_forecast() + assert isinstance(df, pd.DataFrame) + + def test_get_supply_demand_returns_dataframe(self, mixin_instance): + """Test get_supply_demand returns DataFrame.""" + df = mixin_instance.get_supply_demand() + assert isinstance(df, pd.DataFrame) + + def test_get_daily_prices_returns_dataframe(self, mixin_instance): + """Test get_daily_prices returns DataFrame.""" + df = mixin_instance.get_daily_prices() + assert isinstance(df, pd.DataFrame) + + +class TestGridConditionFromString: + """Tests for GridCondition.from_string method.""" + + def test_from_string_normal(self): + """Test parsing normal condition.""" + assert GridCondition.from_string("normal") == GridCondition.NORMAL + assert GridCondition.from_string("Normal Operations") == GridCondition.NORMAL + + def test_from_string_conservation(self): + """Test parsing conservation condition.""" + assert GridCondition.from_string("conservation") == GridCondition.CONSERVATION + assert ( + GridCondition.from_string("conservation appeal") + == GridCondition.CONSERVATION + ) + + def test_from_string_watch(self): + """Test parsing watch condition.""" + assert GridCondition.from_string("watch") == GridCondition.WATCH + assert GridCondition.from_string("weather watch") == GridCondition.WATCH + + def test_from_string_advisory(self): + """Test parsing advisory condition.""" + assert GridCondition.from_string("advisory") == GridCondition.ADVISORY + assert ( + GridCondition.from_string("operating condition notice") + == GridCondition.ADVISORY + ) + + def test_from_string_emergency(self): + """Test parsing emergency condition.""" + assert GridCondition.from_string("emergency") == GridCondition.EMERGENCY + + def test_from_string_eea_levels(self): + """Test parsing EEA levels.""" + assert GridCondition.from_string("eea1") == GridCondition.EEA1 + assert GridCondition.from_string("EEA 1") == GridCondition.EEA1 + assert ( + GridCondition.from_string("energy emergency alert 1") == GridCondition.EEA1 + ) + assert GridCondition.from_string("eea2") == GridCondition.EEA2 + assert GridCondition.from_string("eea3") == GridCondition.EEA3 + + def test_from_string_unknown(self): + """Test parsing unknown condition.""" + assert GridCondition.from_string("unknown_value") == GridCondition.UNKNOWN + assert GridCondition.from_string(None) == GridCondition.UNKNOWN + assert GridCondition.from_string("") == GridCondition.UNKNOWN + + +class TestSafeFloat: + """Tests for _safe_float helper function.""" + + def test_safe_float_valid_float(self): + """Test with valid float.""" + assert _safe_float(3.14) == 3.14 + + def test_safe_float_valid_int(self): + """Test with valid int.""" + assert _safe_float(42) == 42.0 + + def test_safe_float_valid_string(self): + """Test with valid numeric string.""" + assert _safe_float("3.14") == 3.14 + + def test_safe_float_none_returns_default(self): + """Test None returns default.""" + assert _safe_float(None) == 0.0 + assert _safe_float(None, default=99.0) == 99.0 + + def test_safe_float_invalid_returns_default(self): + """Test invalid value returns default.""" + assert _safe_float("not a number") == 0.0 + assert _safe_float({}) == 0.0 + assert _safe_float([]) == 0.0 + + +class TestParseTimestamp: + """Tests for _parse_timestamp helper function.""" + + def test_parse_timestamp_none_returns_now(self): + """Test None returns current timestamp.""" + result = _parse_timestamp(None) + assert isinstance(result, pd.Timestamp) + assert result.tzinfo is not None + + def test_parse_timestamp_epoch_ms(self): + """Test parsing epoch milliseconds.""" + # 2024-01-01 00:00:00 UTC in milliseconds + epoch_ms = 1704067200000 + result = _parse_timestamp(epoch_ms) + assert isinstance(result, pd.Timestamp) + + def test_parse_timestamp_epoch_s(self): + """Test parsing epoch seconds.""" + epoch_s = 1704067200 + result = _parse_timestamp(epoch_s) + assert isinstance(result, pd.Timestamp) + + def test_parse_timestamp_string(self): + """Test parsing string timestamp.""" + result = _parse_timestamp("2024-01-01T12:00:00") + assert isinstance(result, pd.Timestamp) + + def test_parse_timestamp_invalid_returns_now(self): + """Test invalid value returns current timestamp.""" + result = _parse_timestamp("not a date") + assert isinstance(result, pd.Timestamp) + + +class TestFetchJson: + """Tests for _fetch_json helper function.""" + + @patch("tinygrid.ercot.dashboard.httpx.Client") + def test_fetch_json_success(self, mock_client_class): + """Test successful JSON fetch.""" + from unittest.mock import MagicMock + + mock_response = MagicMock() + mock_response.json.return_value = {"data": "test"} + mock_client = MagicMock() + mock_client.get.return_value = mock_response + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + result = _fetch_json("https://example.com/api") + + assert result == {"data": "test"} + + @patch("tinygrid.ercot.dashboard.httpx.Client") + def test_fetch_json_timeout(self, mock_client_class): + """Test timeout returns None.""" + from unittest.mock import MagicMock + + import httpx + + mock_client = MagicMock() + mock_client.get.side_effect = httpx.TimeoutException("Timeout") + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + result = _fetch_json("https://example.com/api") + + assert result is None + + @patch("tinygrid.ercot.dashboard.httpx.Client") + def test_fetch_json_http_error(self, mock_client_class): + """Test HTTP error returns None.""" + from unittest.mock import MagicMock + + import httpx + + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Server Error", + request=MagicMock(), + response=mock_response, + ) + mock_client = MagicMock() + mock_client.get.return_value = mock_response + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + result = _fetch_json("https://example.com/api") + + assert result is None + + +class TestFuelMixEntry: + """Tests for FuelMixEntry dataclass.""" + + def test_create_fuel_mix_entry(self): + """Test creating FuelMixEntry.""" + ts = pd.Timestamp.now(tz="US/Central") + entry = FuelMixEntry( + fuel_type="natural_gas", + generation_mw=25000.0, + percentage=45.5, + timestamp=ts, + ) + + assert entry.fuel_type == "natural_gas" + assert entry.generation_mw == 25000.0 + assert entry.percentage == 45.5 + assert entry.timestamp == ts + + +class TestRenewableStatus: + """Tests for RenewableStatus dataclass.""" + + def test_create_renewable_status(self): + """Test creating RenewableStatus.""" + ts = pd.Timestamp.now(tz="US/Central") + status = RenewableStatus( + wind_mw=18000.0, + solar_mw=8000.0, + wind_forecast_mw=19000.0, + solar_forecast_mw=7500.0, + wind_capacity_mw=35000.0, + solar_capacity_mw=20000.0, + timestamp=ts, + ) + + assert status.wind_mw == 18000.0 + assert status.solar_mw == 8000.0 + assert status.wind_forecast_mw == 19000.0 + assert status.additional_data == {} + + def test_renewable_status_with_additional_data(self): + """Test RenewableStatus with additional data.""" + ts = pd.Timestamp.now(tz="US/Central") + status = RenewableStatus( + wind_mw=18000.0, + solar_mw=8000.0, + wind_forecast_mw=19000.0, + solar_forecast_mw=7500.0, + wind_capacity_mw=35000.0, + solar_capacity_mw=20000.0, + timestamp=ts, + additional_data={"extra": "data"}, + ) + + assert status.additional_data == {"extra": "data"} + + +class TestDashboardWithMocking: + """Tests for dashboard methods with mocked HTTP responses.""" + + @pytest.fixture + def mixin_instance(self): + """Create a test instance with the mixin.""" + + class TestClass(ERCOTDashboardMixin): + pass + + return TestClass() + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_success(self, mock_fetch, mixin_instance): + """Test get_status with successful response.""" + mock_fetch.return_value = { + "current": { + "condition": "normal", + "demand": 50000, + "capacity": 70000, + "reserves": 20000, + "lastUpdated": 1704067200000, + } + } + + status = mixin_instance.get_status() + + assert status.condition == GridCondition.NORMAL + assert status.current_load == 50000.0 + assert status.capacity == 70000.0 + assert status.reserves == 20000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_calculates_reserves(self, mock_fetch, mixin_instance): + """Test get_status calculates reserves when not provided.""" + mock_fetch.return_value = { + "current": { + "condition": "normal", + "demand": 50000, + "capacity": 70000, + } + } + + status = mixin_instance.get_status() + + # Reserves should be calculated as capacity - load + assert status.reserves == 20000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_failure(self, mock_fetch, mixin_instance): + """Test get_status returns unavailable on failure.""" + mock_fetch.return_value = None + + status = mixin_instance.get_status() + assert status.condition == GridCondition.UNKNOWN assert "not available" in status.message - def test_get_fuel_mix_returns_empty(self, mixin_instance): - """Test get_fuel_mix returns empty DataFrame.""" + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_success(self, mock_fetch, mixin_instance): + """Test get_fuel_mix with successful response.""" + mock_fetch.return_value = { + "data": [ + {"fuel": "gas", "gen": 25000, "percent": 45}, + {"fuel": "wind", "gen": 18000, "percent": 32}, + ], + "lastUpdated": 1704067200000, + } + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) - assert df.empty + assert len(df) == 2 + assert "fuel_type" in df.columns + assert "generation_mw" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_calculates_percentage(self, mock_fetch, mixin_instance): + """Test get_fuel_mix calculates percentage when not provided.""" + mock_fetch.return_value = { + "data": [ + {"fuel": "gas", "gen": 50000}, + {"fuel": "wind", "gen": 50000}, + ], + "lastUpdated": 1704067200000, + } + + df = mixin_instance.get_fuel_mix() + + # Each should be 50% + assert df["percentage"].iloc[0] == 50.0 + assert df["percentage"].iloc[1] == 50.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_as_list(self, mock_fetch, mixin_instance): + """Test get_fuel_mix returns list when as_dataframe=False.""" + mock_fetch.return_value = { + "data": [ + {"fuel": "gas", "gen": 25000, "percent": 45}, + ], + "lastUpdated": 1704067200000, + } + + result = mixin_instance.get_fuel_mix(as_dataframe=False) + + assert isinstance(result, list) + assert len(result) == 1 + assert isinstance(result[0], FuelMixEntry) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_renewable_generation_success(self, mock_fetch, mixin_instance): + """Test get_renewable_generation with successful response.""" + mock_fetch.return_value = { + "current": { + "windActual": 18000, + "solarActual": 8000, + "windForecast": 19000, + "solarForecast": 7500, + "windCapacity": 35000, + "solarCapacity": 20000, + }, + "lastUpdated": 1704067200000, + } + + status = mixin_instance.get_renewable_generation() + + assert isinstance(status, RenewableStatus) + assert status.wind_mw == 18000.0 + assert status.solar_mw == 8000.0 + assert status.wind_forecast_mw == 19000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_supply_demand_success(self, mock_fetch, mixin_instance): + """Test get_supply_demand with successful response.""" + mock_fetch.return_value = { + "data": [ + {"hour": 1, "demand": 45000, "supply": 60000, "reserves": 15000}, + {"hour": 2, "demand": 46000, "supply": 60000, "reserves": 14000}, + ], + "lastUpdated": 1704067200000, + } + + df = mixin_instance.get_supply_demand() - def test_get_fuel_mix_with_date_param(self, mixin_instance): - """Test get_fuel_mix accepts date parameter.""" - df = mixin_instance.get_fuel_mix(date="yesterday") assert isinstance(df, pd.DataFrame) - assert df.empty + assert len(df) == 2 + assert "hour" in df.columns + assert "demand" in df.columns + assert "supply" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_daily_prices_success(self, mock_fetch, mixin_instance): + """Test get_daily_prices with successful response.""" + mock_fetch.return_value = { + "data": [ + {"settlementPoint": "HB_HOUSTON", "price": 25.50}, + {"settlementPoint": "HB_NORTH", "price": 24.00}, + ], + "lastUpdated": 1704067200000, + } + + df = mixin_instance.get_daily_prices() - def test_get_energy_storage_resources_returns_empty(self, mixin_instance): - """Test get_energy_storage_resources returns empty DataFrame.""" - df = mixin_instance.get_energy_storage_resources() assert isinstance(df, pd.DataFrame) - assert df.empty + assert len(df) == 2 + assert "settlement_point" in df.columns + assert "price" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_system_wide_demand_success(self, mock_fetch, mixin_instance): + """Test get_system_wide_demand with successful response.""" + mock_fetch.return_value = { + "current": { + "demand": 50000, + "capacity": 70000, + "reserves": 20000, + }, + "hourly": [ + {"hour": 1, "demand": 45000, "capacity": 70000, "reserves": 25000}, + ], + "lastUpdated": 1704067200000, + } - def test_get_system_wide_demand_returns_empty(self, mixin_instance): - """Test get_system_wide_demand returns empty DataFrame.""" df = mixin_instance.get_system_wide_demand() + assert isinstance(df, pd.DataFrame) - assert df.empty + assert len(df) == 2 # current + 1 hourly + assert "hour" in df.columns + assert "demand" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_energy_storage_with_data(self, mock_fetch, mixin_instance): + """Test get_energy_storage_resources with ESR data.""" + mock_fetch.return_value = { + "current": { + "esr": { + "charging": 500, + "discharging": 1000, + "net": 500, + "capacity": 3000, + } + }, + "lastUpdated": 1704067200000, + } + + df = mixin_instance.get_energy_storage_resources() - def test_get_renewable_generation_returns_empty(self, mixin_instance): - """Test get_renewable_generation returns empty DataFrame.""" - df = mixin_instance.get_renewable_generation() assert isinstance(df, pd.DataFrame) - assert df.empty + if len(df) > 0: + assert "charging_mw" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_capacity_committed_success(self, mock_fetch, mixin_instance): + """Test get_capacity_committed with successful response.""" + mock_fetch.return_value = { + "data": [ + {"hour": 1, "committed": 60000, "available": 70000}, + {"hour": 2, "committed": 61000, "available": 70000}, + ], + "lastUpdated": 1704067200000, + } - def test_get_capacity_committed_returns_empty(self, mixin_instance): - """Test get_capacity_committed returns empty DataFrame.""" df = mixin_instance.get_capacity_committed() - assert isinstance(df, pd.DataFrame) - assert df.empty - def test_get_capacity_forecast_returns_empty(self, mixin_instance): - """Test get_capacity_forecast returns empty DataFrame.""" - df = mixin_instance.get_capacity_forecast() assert isinstance(df, pd.DataFrame) - assert df.empty + assert len(df) == 2 + assert "hour" in df.columns diff --git a/tests/test_polling.py b/tests/test_polling.py new file mode 100644 index 0000000..7940e61 --- /dev/null +++ b/tests/test_polling.py @@ -0,0 +1,414 @@ +"""Tests for tinygrid.ercot.polling module.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd + +from tinygrid.ercot.polling import ( + DEFAULT_POLL_INTERVAL, + MAX_CONSECUTIVE_ERRORS, + MIN_POLL_INTERVAL, + ERCOTPoller, + PollResult, + poll_latest, +) +from tinygrid.errors import GridAPIError, GridError + + +class TestPollingConstants: + """Tests for polling constants.""" + + def test_min_poll_interval(self): + """Test minimum poll interval.""" + assert MIN_POLL_INTERVAL == 2.0 + + def test_default_poll_interval(self): + """Test default poll interval.""" + assert DEFAULT_POLL_INTERVAL == 60.0 + + def test_max_consecutive_errors(self): + """Test max consecutive errors.""" + assert MAX_CONSECUTIVE_ERRORS == 5 + + +class TestPollResult: + """Tests for PollResult dataclass.""" + + def test_create_success_result(self): + """Test creating successful poll result.""" + df = pd.DataFrame({"col": [1, 2, 3]}) + ts = pd.Timestamp.now(tz="US/Central") + + result = PollResult( + data=df, + timestamp=ts, + success=True, + iteration=0, + ) + + assert result.success is True + assert result.error is None + assert result.iteration == 0 + assert len(result.data) == 3 + + def test_create_error_result(self): + """Test creating error poll result.""" + ts = pd.Timestamp.now(tz="US/Central") + error = GridAPIError("API Error", status_code=500) + + result = PollResult( + data=None, + timestamp=ts, + success=False, + error=error, + iteration=5, + ) + + assert result.success is False + assert result.data is None + assert result.error is error + assert result.iteration == 5 + + +class TestERCOTPollerInit: + """Tests for ERCOTPoller initialization.""" + + def test_init_defaults(self): + """Test default initialization.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client) + + assert poller.client is mock_client + assert poller.interval == DEFAULT_POLL_INTERVAL + assert poller.max_errors == MAX_CONSECUTIVE_ERRORS + assert poller.backoff_factor == 2.0 + assert poller.max_backoff == 300.0 + + def test_init_custom_interval(self): + """Test custom interval initialization.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client, interval=30.0) + + assert poller.interval == 30.0 + + def test_init_enforces_min_interval(self): + """Test that minimum interval is enforced.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client, interval=0.5) + + assert poller.interval == MIN_POLL_INTERVAL + + def test_init_custom_max_errors(self): + """Test custom max errors.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client, max_errors=10) + + assert poller.max_errors == 10 + + def test_init_custom_backoff(self): + """Test custom backoff settings.""" + mock_client = MagicMock() + poller = ERCOTPoller( + client=mock_client, + backoff_factor=3.0, + max_backoff=600.0, + ) + + assert poller.backoff_factor == 3.0 + assert poller.max_backoff == 600.0 + + +class TestERCOTPollerPollOnce: + """Tests for ERCOTPoller._poll_once method.""" + + def test_poll_once_success(self): + """Test successful single poll.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame({"col": [1, 2]})) + + poller = ERCOTPoller(client=mock_client) + result = poller._poll_once(mock_method, iteration=0) + + assert result.success is True + assert len(result.data) == 2 + assert result.iteration == 0 + + def test_poll_once_with_kwargs(self): + """Test single poll passes kwargs.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client) + poller._poll_once(mock_method, iteration=0, param1="value1") + + mock_method.assert_called_once() + call_kwargs = mock_method.call_args[1] + assert call_kwargs["param1"] == "value1" + + def test_poll_once_adds_default_start(self): + """Test that poll_once adds default start if not provided.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client) + poller._poll_once(mock_method, iteration=0) + + call_kwargs = mock_method.call_args[1] + assert call_kwargs["start"] == "today" + + def test_poll_once_grid_error(self): + """Test poll_once handles GridError.""" + mock_client = MagicMock() + mock_method = MagicMock(side_effect=GridAPIError("API Error", status_code=500)) + + poller = ERCOTPoller(client=mock_client) + result = poller._poll_once(mock_method, iteration=0) + + assert result.success is False + assert result.data is None + assert isinstance(result.error, GridError) + + def test_poll_once_generic_exception(self): + """Test poll_once handles generic exceptions.""" + mock_client = MagicMock() + mock_method = MagicMock(side_effect=Exception("Unexpected error")) + + poller = ERCOTPoller(client=mock_client) + result = poller._poll_once(mock_method, iteration=0) + + assert result.success is False + assert result.data is None + + +class TestERCOTPollerBackoff: + """Tests for ERCOTPoller backoff handling.""" + + def test_handle_error_increments_count(self): + """Test that _handle_error increments error count.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client) + + assert poller._consecutive_errors == 0 + + poller._handle_error() + + assert poller._consecutive_errors == 1 + + def test_handle_error_increases_backoff(self): + """Test that _handle_error increases backoff.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client, interval=10.0) + + assert poller._current_backoff == 0.0 + + poller._handle_error() + + assert poller._current_backoff > 0 + + def test_handle_error_respects_max_backoff(self): + """Test that backoff is capped at max_backoff.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client, max_backoff=10.0) + + # Simulate many errors + for _ in range(20): + poller._handle_error() + + assert poller._current_backoff <= 10.0 + + def test_reset_backoff(self): + """Test that _reset_backoff resets state.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client) + + # Simulate some errors + poller._handle_error() + poller._handle_error() + + assert poller._consecutive_errors > 0 + assert poller._current_backoff > 0 + + poller._reset_backoff() + + assert poller._consecutive_errors == 0 + assert poller._current_backoff == 0.0 + + +class TestERCOTPollerStop: + """Tests for ERCOTPoller.stop method.""" + + def test_stop_sets_running_false(self): + """Test that stop sets _running to False.""" + mock_client = MagicMock() + poller = ERCOTPoller(client=mock_client) + + poller._running = True + poller.stop() + + assert poller._running is False + + +class TestERCOTPollerPollIter: + """Tests for ERCOTPoller.poll_iter method.""" + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_yields_results(self, mock_sleep): + """Test poll_iter yields PollResult objects.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame({"col": [1]})) + + poller = ERCOTPoller(client=mock_client) + + results = list(poller.poll_iter(method=mock_method, max_iterations=3)) + + assert len(results) == 3 + assert all(isinstance(r, PollResult) for r in results) + assert all(r.success for r in results) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_respects_max_iterations(self, mock_sleep): + """Test poll_iter stops at max_iterations.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client) + + results = list(poller.poll_iter(method=mock_method, max_iterations=5)) + + assert len(results) == 5 + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_stops_on_max_errors(self, mock_sleep): + """Test poll_iter stops after max consecutive errors.""" + mock_client = MagicMock() + mock_method = MagicMock(side_effect=GridAPIError("Error", status_code=500)) + + poller = ERCOTPoller(client=mock_client, max_errors=3) + + results = list(poller.poll_iter(method=mock_method, max_iterations=10)) + + # Should stop after 3 errors + assert len(results) == 3 + assert all(not r.success for r in results) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_resets_backoff_on_success(self, mock_sleep): + """Test poll_iter resets backoff after success.""" + mock_client = MagicMock() + # First call fails, second succeeds + mock_method = MagicMock( + side_effect=[ + GridAPIError("Error", status_code=500), + pd.DataFrame({"col": [1]}), + ] + ) + + poller = ERCOTPoller(client=mock_client) + + results = list(poller.poll_iter(method=mock_method, max_iterations=2)) + + assert len(results) == 2 + assert not results[0].success + assert results[1].success + # Backoff should be reset after success + assert poller._consecutive_errors == 0 + + +class TestERCOTPollerPollCallback: + """Tests for ERCOTPoller.poll method with callback.""" + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_calls_callback(self, mock_sleep): + """Test poll calls callback for each iteration.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame({"col": [1]})) + callback_results = [] + + def callback(result): + callback_results.append(result) + + poller = ERCOTPoller(client=mock_client) + poller.poll(method=mock_method, callback=callback, max_iterations=3) + + assert len(callback_results) == 3 + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_stops_on_max_iterations(self, mock_sleep): + """Test poll stops at max_iterations.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + callback_count = [0] + + def callback(result): + callback_count[0] += 1 + + poller = ERCOTPoller(client=mock_client) + poller.poll(method=mock_method, callback=callback, max_iterations=5) + + assert callback_count[0] == 5 + + +class TestPollLatest: + """Tests for poll_latest convenience function.""" + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_latest_yields_dataframes(self, mock_sleep): + """Test poll_latest yields DataFrames.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame({"col": [1, 2]})) + + results = list( + poll_latest( + client=mock_client, + method=mock_method, + interval=60, + max_iterations=3, + ) + ) + + assert len(results) == 3 + assert all(isinstance(r, pd.DataFrame) for r in results) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_latest_skips_failures(self, mock_sleep): + """Test poll_latest skips failed polls.""" + mock_client = MagicMock() + # First fails, second succeeds + mock_method = MagicMock( + side_effect=[ + GridAPIError("Error", status_code=500), + pd.DataFrame({"col": [1]}), + ] + ) + + results = list( + poll_latest( + client=mock_client, + method=mock_method, + max_iterations=2, + ) + ) + + # Only successful poll should be yielded + assert len(results) == 1 + assert isinstance(results[0], pd.DataFrame) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_latest_passes_kwargs(self, mock_sleep): + """Test poll_latest passes kwargs to method.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + list( + poll_latest( + client=mock_client, + method=mock_method, + max_iterations=1, + custom_param="value", + ) + ) + + call_kwargs = mock_method.call_args[1] + assert call_kwargs["custom_param"] == "value" diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.py new file mode 100644 index 0000000..50bbbbe --- /dev/null +++ b/tests/test_rate_limiter.py @@ -0,0 +1,316 @@ +"""Tests for tinygrid.utils.rate_limiter module.""" + +from __future__ import annotations + +import time + +import pytest + +from tinygrid.utils.rate_limiter import ( + ERCOT_REQUESTS_PER_MINUTE, + AsyncRateLimiter, + RateLimiter, + rate_limited, +) + + +class TestRateLimiterConstants: + """Tests for rate limiter constants.""" + + def test_ercot_requests_per_minute(self): + """Test ERCOT rate limit constant.""" + assert ERCOT_REQUESTS_PER_MINUTE == 30 + + +class TestRateLimiter: + """Tests for RateLimiter class.""" + + def test_initialization_defaults(self): + """Test default initialization.""" + limiter = RateLimiter() + assert limiter.requests_per_minute == 30 + assert limiter.burst_size == 30 + assert limiter.available_tokens == 30 + + def test_initialization_custom_rate(self): + """Test custom rate initialization.""" + limiter = RateLimiter(requests_per_minute=60) + assert limiter.requests_per_minute == 60 + assert limiter.burst_size == 60 + + def test_initialization_custom_burst_size(self): + """Test custom burst size initialization.""" + limiter = RateLimiter(requests_per_minute=30, burst_size=10) + assert limiter.burst_size == 10 + assert limiter.available_tokens == 10 + + def test_min_interval(self): + """Test min_interval property.""" + limiter = RateLimiter(requests_per_minute=60) + assert limiter.min_interval == 1.0 # 60 seconds / 60 requests + + limiter = RateLimiter(requests_per_minute=30) + assert limiter.min_interval == 2.0 # 60 seconds / 30 requests + + def test_acquire_consumes_token(self): + """Test that acquire consumes a token.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=10) + initial_tokens = limiter.available_tokens + + limiter.acquire() + + assert limiter.available_tokens < initial_tokens + + def test_acquire_burst(self): + """Test acquiring multiple tokens in burst.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=5) + + # Should be able to acquire 5 tokens immediately + for _ in range(5): + result = limiter.acquire(timeout=0.01) + assert result is True + + def test_acquire_with_timeout_fails_when_empty(self): + """Test that acquire with timeout returns False when no tokens.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=1) + limiter.acquire() # Consume the only token + + # Should timeout waiting for another token + result = limiter.acquire(timeout=0.01) + assert result is False + + def test_tokens_refill_over_time(self): + """Test that tokens refill over time.""" + limiter = RateLimiter(requests_per_minute=600, burst_size=10) # 10/second + + # Consume all tokens + for _ in range(10): + limiter.acquire() + + assert limiter.available_tokens < 1 + + # Wait for refill + time.sleep(0.2) + + # Should have refilled some tokens + assert limiter.available_tokens >= 1 + + def test_context_manager(self): + """Test context manager usage.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=10) + initial = limiter.available_tokens + + with limiter: + pass + + assert limiter.available_tokens < initial + + def test_release_is_noop(self): + """Test that release does nothing (token bucket pattern).""" + limiter = RateLimiter(requests_per_minute=60, burst_size=10) + limiter.acquire() + + # Store count, call release immediately + initial_count = limiter._tokens # Use internal state, not property + limiter.release() + + # Release should not change token count (internal state unchanged) + assert limiter._tokens == initial_count + + def test_reset(self): + """Test reset restores full capacity.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=10) + + # Consume some tokens + for _ in range(5): + limiter.acquire() + + limiter.reset() + + assert limiter.available_tokens == 10 + + +class TestAsyncRateLimiter: + """Tests for AsyncRateLimiter class.""" + + def test_initialization_defaults(self): + """Test default initialization.""" + limiter = AsyncRateLimiter() + assert limiter.requests_per_minute == 30 + assert limiter.burst_size == 30 + + def test_initialization_custom(self): + """Test custom initialization.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=10) + assert limiter.requests_per_minute == 60 + assert limiter.burst_size == 10 + + @pytest.mark.asyncio + async def test_acquire_consumes_token(self): + """Test that acquire consumes a token.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=10) + initial = limiter.available_tokens + + await limiter.acquire() + + assert limiter.available_tokens < initial + + @pytest.mark.asyncio + async def test_acquire_burst(self): + """Test acquiring multiple tokens in burst.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=5) + + # Should be able to acquire 5 tokens immediately + for _ in range(5): + result = await limiter.acquire(timeout=0.01) + assert result is True + + @pytest.mark.asyncio + async def test_acquire_with_timeout_fails_when_empty(self): + """Test that acquire with timeout returns False when no tokens.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=1) + await limiter.acquire() # Consume the only token + + # Should timeout waiting for another token + result = await limiter.acquire(timeout=0.01) + assert result is False + + @pytest.mark.asyncio + async def test_async_context_manager(self): + """Test async context manager usage.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=10) + initial = limiter.available_tokens + + async with limiter: + pass + + assert limiter.available_tokens < initial + + @pytest.mark.asyncio + async def test_release_is_noop(self): + """Test that release does nothing.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=10) + await limiter.acquire() + + # Store internal count, call release immediately + initial_count = limiter._tokens # Use internal state + await limiter.release() + + # Release should not change token count + assert limiter._tokens == initial_count + + def test_reset(self): + """Test reset restores full capacity.""" + limiter = AsyncRateLimiter(requests_per_minute=60, burst_size=10) + + # Manually reduce tokens + limiter._tokens = 2 + + limiter.reset() + + assert limiter.available_tokens == 10 + + +class TestRateLimitedDecorator: + """Tests for rate_limited decorator.""" + + def test_decorator_with_default_limiter(self): + """Test decorator creates default limiter.""" + call_count = 0 + + @rate_limited() + def my_func(): + nonlocal call_count + call_count += 1 + return "result" + + result = my_func() + + assert result == "result" + assert call_count == 1 + + def test_decorator_with_custom_rate(self): + """Test decorator with custom rate.""" + + @rate_limited(requests_per_minute=60) + def my_func(): + return "result" + + result = my_func() + assert result == "result" + + def test_decorator_with_existing_limiter(self): + """Test decorator with existing limiter.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=10) + initial = limiter.available_tokens + + @rate_limited(limiter=limiter) + def my_func(): + return "result" + + result = my_func() + + assert result == "result" + assert limiter.available_tokens < initial + + def test_decorator_preserves_function_metadata(self): + """Test decorator preserves function name and docstring.""" + + @rate_limited() + def my_documented_func(): + """This is my docstring.""" + return "result" + + assert my_documented_func.__name__ == "my_documented_func" + assert my_documented_func.__doc__ == """This is my docstring.""" + + def test_decorator_passes_arguments(self): + """Test decorator passes args and kwargs correctly.""" + + @rate_limited() + def add(a, b, c=0): + return a + b + c + + assert add(1, 2) == 3 + assert add(1, 2, c=3) == 6 + + +class TestRateLimiterEdgeCases: + """Edge case tests for rate limiter.""" + + def test_very_high_rate(self): + """Test with very high rate limit.""" + limiter = RateLimiter(requests_per_minute=6000) # 100/second + assert limiter.min_interval == 0.01 + + def test_very_low_rate(self): + """Test with very low rate limit.""" + limiter = RateLimiter(requests_per_minute=1) + assert limiter.min_interval == 60.0 + + def test_fractional_burst_size(self): + """Test with fractional burst size.""" + limiter = RateLimiter(requests_per_minute=60, burst_size=1.5) + assert limiter.burst_size == 1.5 + + def test_concurrent_acquire(self): + """Test thread safety with concurrent acquires.""" + import threading + + limiter = RateLimiter(requests_per_minute=600, burst_size=100) + results = [] + + def acquire_token(): + result = limiter.acquire(timeout=1.0) + results.append(result) + + threads = [threading.Thread(target=acquire_token) for _ in range(50)] + + for t in threads: + t.start() + for t in threads: + t.join() + + # All should succeed since we have 100 burst capacity + assert all(results) + assert len(results) == 50 diff --git a/tinygrid/README.md b/tinygrid/README.md index 0c22e95..780d665 100644 --- a/tinygrid/README.md +++ b/tinygrid/README.md @@ -10,16 +10,18 @@ The tinygrid SDK uses a modular mixin-based architecture for the ERCOT client: tinygrid/ ├── ercot/ # ERCOT client package │ ├── __init__.py # Main ERCOT class (combines all mixins) -│ ├── client.py # ERCOTBase - auth, retry, pagination, core helpers +│ ├── client.py # ERCOTBase - auth, retry, pagination, rate limiting │ ├── endpoints.py # ERCOTEndpointsMixin - 100+ pyercot endpoint wrappers │ ├── api.py # ERCOTAPIMixin - high-level unified API methods │ ├── archive.py # ERCOTArchive - historical archive API access -│ ├── dashboard.py # ERCOTDashboardMixin - public dashboard methods (no auth) +│ ├── dashboard.py # ERCOTDashboardMixin - public dashboard JSON endpoints │ ├── documents.py # ERCOTDocumentsMixin - MIS document fetching +│ ├── eia.py # EIAClient - EIA API integration +│ ├── polling.py # ERCOTPoller - real-time polling utilities │ └── transforms.py # Data filtering and transformation utilities ├── auth/ # Authentication handling ├── constants/ # Market types, location enums, mappings -├── utils/ # Date parsing, timezone handling, decorators +├── utils/ # Date parsing, timezone, decorators, rate limiting └── errors.py # Exception hierarchy ``` @@ -67,21 +69,31 @@ df = ercot.get_as_prices(start="today") ### Dashboard Module (dashboard.py) -**Note:** The dashboard methods are placeholders. ERCOT does not provide -documented public JSON endpoints for dashboard data. Use authenticated -API methods instead: +Access ERCOT's public dashboard JSON endpoints (no auth required): ```python ercot = ERCOT() -# For system load data, use: -load = ercot.get_load(start="today", by="weather_zone") +# Get current grid status +status = ercot.get_status() +print(f"Condition: {status.condition}") # NORMAL, WATCH, EMERGENCY, EEA1, etc. +print(f"Load: {status.current_load:,.0f} MW") +print(f"Reserves: {status.reserves:,.0f} MW") -# For forecasts, use: -wind = ercot.get_wind_forecast(start="today") -solar = ercot.get_solar_forecast(start="today") +# Get fuel mix +fuel_mix = ercot.get_fuel_mix() # Returns DataFrame + +# Get renewable generation +renewables = ercot.get_renewable_generation() # Returns RenewableStatus +print(f"Wind: {renewables.wind_mw:,.0f} MW") +print(f"Solar: {renewables.solar_mw:,.0f} MW") + +# Get supply/demand curve +supply_demand = ercot.get_supply_demand() ``` +**Note:** These endpoints are undocumented and may change without notice. + ### Historical Yearly Data (documents.py) Access full-year historical data from ERCOT's MIS document system: @@ -127,6 +139,7 @@ with ERCOT(auth=auth) as ercot: Core functionality inherited by ERCOT class: - Authentication and token management - Retry with exponential backoff (via tenacity) +- Rate limiting (30 req/min, configurable) - Pagination handling for large result sets - DataFrame conversion from API responses - Historical data routing decisions @@ -161,6 +174,38 @@ MIS document fetching for yearly historical data: - `get_settlement_point_mapping()` - Access to ERCOT's Market Information System reports +### eia.py (EIAClient) + +Access ERCOT data via the EIA API (useful for data before Dec 2023): + +```python +from tinygrid.ercot import EIAClient + +eia = EIAClient(api_key="your-eia-key") + +# Get hourly demand (from 2019) +demand = eia.get_demand(start="2022-01-01", end="2022-01-07") + +# Get generation by fuel type +gen = eia.get_generation_by_fuel(start="2022-01-01") +``` + +### polling.py (ERCOTPoller) + +Real-time polling utilities for continuous data monitoring: + +```python +from tinygrid.ercot import ERCOTPoller, poll_latest + +# Simple generator pattern +for df in poll_latest(ercot, ercot.get_spp, interval=60, max_iterations=10): + process(df) + +# Callback pattern with error handling +poller = ERCOTPoller(client=ercot, interval=60, max_errors=5) +poller.poll(method=ercot.get_spp, callback=handle_data) +``` + ### transforms.py Standalone data transformation functions: @@ -184,4 +229,4 @@ Standalone data transformation functions: pytest tests/ ``` -505 tests covering all functionality. +746 tests with 95% code coverage. diff --git a/tinygrid/constants/ercot.py b/tinygrid/constants/ercot.py index 32a55d3..83e2e6b 100644 --- a/tinygrid/constants/ercot.py +++ b/tinygrid/constants/ercot.py @@ -32,6 +32,51 @@ def __str__(self): PUBLIC_API_BASE_URL = "https://api.ercot.com/api/public-reports" ESR_API_BASE_URL = "https://api.ercot.com/api/public-data" +# Developer resources +DEVELOPER_PORTAL_URL = "https://developer.ercot.com" +API_EXPLORER_URL = "https://apiexplorer.ercot.com" +OPENAPI_SPECS_URL = "https://github.com/ercot/api-specs" + +# ============================================================================ +# API Data Availability - Important Limitations +# ============================================================================ +# +# The ERCOT Public API has the following key limitations that users should +# be aware of when building applications: +# +# 1. API DATA START DATE: December 11, 2023 +# - The REST API only contains data from this date forward +# - For earlier data, use MIS document downloads or the archive API +# - Historical archives extend 7+ years back +# +# 2. DATA DELAY: Approximately 1 hour +# - Real-time data is NOT truly real-time +# - There is approximately a 1-hour delay from actual grid operations +# to data availability in the API +# +# 3. GEOGRAPHIC RESTRICTION: US IP addresses only +# - The API blocks requests from non-US IP addresses +# - Users outside the US will need a VPN or US-based proxy +# +# 4. RATE LIMIT: 30 requests per minute +# - Exceeding this limit results in HTTP 429 errors +# - Use the built-in rate limiter (enabled by default) +# +# 5. FILE LIMIT: 1,000 documents per bulk download +# - When using archive bulk downloads, max 1,000 files per request + +# Date when ERCOT's Public API became available +API_LAUNCH_DATE = "2023-12-11" + +# Approximate delay (in minutes) from real-time to API availability +API_DATA_DELAY_MINUTES = 60 + +# Maximum documents per bulk archive download +MAX_BULK_DOWNLOAD_FILES = 1000 + +# Rate limit (requests per minute) +API_RATE_LIMIT = 30 + class Market(StrEnum): """ERCOT market types for price data.""" @@ -145,13 +190,21 @@ class SettlementPointType(StrEnum): "np4-188-cd": "np4-188-cd", # DAM AS MCPC prices "np4-33-cd": "np4-33-cd", # DAM AS plan # Forecasts - "np4-732-cd": "np4-732-cd", # Wind forecast - "np4-742-cd": "np4-742-cd", # Wind forecast geo - "np4-737-cd": "np4-737-cd", # Solar forecast - "np4-745-cd": "np4-745-cd", # Solar forecast geo + "np4-732-cd": "np4-732-cd", # Wind forecast hourly + "np4-733-cd": "np4-733-cd", # Wind 5-minute averaged + "np4-742-cd": "np4-742-cd", # Wind forecast geo hourly + "np4-743-cd": "np4-743-cd", # Wind 5-minute geo + "np4-737-cd": "np4-737-cd", # Solar forecast hourly + "np4-738-cd": "np4-738-cd", # Solar 5-minute averaged + "np4-745-cd": "np4-745-cd", # Solar forecast geo hourly + "np4-746-cd": "np4-746-cd", # Solar 5-minute geo # Load "np6-345-cd": "np6-345-cd", # Load by weather zone "np6-346-cd": "np6-346-cd", # Load by forecast zone + # System-wide / Transmission + "np6-625-cd": "np6-625-cd", # Total ERCOT generation + "np6-626-cd": "np6-626-cd", # DC tie flows + "np6-235-cd": "np6-235-cd", # System-wide actuals } # Column name mappings for standardization (raw API name -> user-friendly name) diff --git a/tinygrid/ercot/__init__.py b/tinygrid/ercot/__init__.py index 9a12867..4a64af9 100644 --- a/tinygrid/ercot/__init__.py +++ b/tinygrid/ercot/__init__.py @@ -156,9 +156,17 @@ from .api import ERCOTAPIMixin from .archive import ERCOTArchive from .client import ERCOTBase -from .dashboard import ERCOTDashboardMixin, GridCondition, GridStatus +from .dashboard import ( + ERCOTDashboardMixin, + FuelMixEntry, + GridCondition, + GridStatus, + RenewableStatus, +) from .documents import REPORT_TYPE_IDS, ERCOTDocumentsMixin +from .eia import EIAClient from .endpoints import ERCOTEndpointsMixin +from .polling import ERCOTPoller, PollResult, poll_latest @define @@ -209,6 +217,10 @@ class ERCOT( retry_max_wait: Maximum wait time between retries in seconds. Defaults to 60.0. page_size: Number of records per page when fetching data. Defaults to 10000. max_concurrent_requests: Maximum number of concurrent page requests. Defaults to 5. + rate_limit_enabled: Whether to enforce rate limiting. Defaults to True. + ERCOT's API has a limit of 30 requests per minute. + requests_per_minute: Maximum requests per minute when rate limiting is enabled. + Defaults to 30 (ERCOT's documented limit). """ pass # All functionality comes from mixins @@ -223,9 +235,20 @@ class ERCOT( "LOAD_ZONES", "REPORT_TYPE_IDS", "TRADING_HUBS", + # EIA integration + "EIAClient", # Archive access "ERCOTArchive", + # Polling utilities + "ERCOTPoller", + # Dashboard types + "FuelMixEntry", + "GridCondition", + "GridStatus", "LocationType", "Market", + "PollResult", + "RenewableStatus", "SettlementPointType", + "poll_latest", ] diff --git a/tinygrid/ercot/api.py b/tinygrid/ercot/api.py index c012c10..199640e 100644 --- a/tinygrid/ercot/api.py +++ b/tinygrid/ercot/api.py @@ -405,6 +405,7 @@ def get_wind_forecast( start: str | pd.Timestamp = "today", end: str | pd.Timestamp | None = None, by_region: bool = False, + resolution: str = "hourly", ) -> pd.DataFrame: """Get wind power production forecast. @@ -412,36 +413,92 @@ def get_wind_forecast( start: Start date end: End date (defaults to start + 1 day) by_region: If True, get by geographical region + resolution: Data resolution - "hourly" (default) or "5min" for 5-minute data Returns: DataFrame with wind forecast data + + Example: + ```python + ercot = ERCOT() + + # Hourly wind forecast (default) + df = ercot.get_wind_forecast(start="2024-01-15") + + # 5-minute wind data + df = ercot.get_wind_forecast(start="2024-01-15", resolution="5min") + + # 5-minute wind data by geographic region + df = ercot.get_wind_forecast( + start="2024-01-15", + resolution="5min", + by_region=True, + ) + ``` """ start_ts, end_ts = parse_date_range(start, end) - if by_region: - if self._needs_historical(start_ts, "forecast"): - df = self._get_archive().fetch_historical( - endpoint="/np4-742-cd/wpp_hrly_actual_fcast_geo", - start=start_ts, - end=end_ts, - ) + # Validate resolution + resolution = resolution.lower() + if resolution not in ("hourly", "5min", "5-min", "5_min"): + raise ValueError( + f"Invalid resolution: {resolution}. Use 'hourly' or '5min'." + ) + + use_5min = resolution in ("5min", "5-min", "5_min") + + if use_5min: + # 5-minute data endpoints + if by_region: + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-743-cd/wpp_actual_5min_avg_values_geo", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_wpp_actual_5min_avg_values_geo( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - df = self.get_wpp_hourly_actual_forecast_geo( - posted_datetime_from=format_api_date(start_ts), - posted_datetime_to=format_api_date(end_ts), - ) + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-733-cd/wpp_actual_5min_avg_values", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_wpp_actual_5min_avg_values( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - if self._needs_historical(start_ts, "forecast"): - df = self._get_archive().fetch_historical( - endpoint="/np4-732-cd/wpp_hrly_avrg_actl_fcast", - start=start_ts, - end=end_ts, - ) + # Hourly data endpoints (default) + if by_region: + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-742-cd/wpp_hrly_actual_fcast_geo", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_wpp_hourly_actual_forecast_geo( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - df = self.get_wpp_hourly_average_actual_forecast( - posted_datetime_from=format_api_date(start_ts), - posted_datetime_to=format_api_date(end_ts), - ) + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-732-cd/wpp_hrly_avrg_actl_fcast", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_wpp_hourly_average_actual_forecast( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) df = filter_by_date(df, start_ts, end_ts, date_column="Posted Datetime") return standardize_columns(df) @@ -451,6 +508,7 @@ def get_solar_forecast( start: str | pd.Timestamp = "today", end: str | pd.Timestamp | None = None, by_region: bool = False, + resolution: str = "hourly", ) -> pd.DataFrame: """Get solar power production forecast. @@ -458,40 +516,224 @@ def get_solar_forecast( start: Start date end: End date (defaults to start + 1 day) by_region: If True, get by geographical region + resolution: Data resolution - "hourly" (default) or "5min" for 5-minute data Returns: DataFrame with solar forecast data + + Example: + ```python + ercot = ERCOT() + + # Hourly solar forecast (default) + df = ercot.get_solar_forecast(start="2024-01-15") + + # 5-minute solar data + df = ercot.get_solar_forecast(start="2024-01-15", resolution="5min") + + # 5-minute solar data by geographic region + df = ercot.get_solar_forecast( + start="2024-01-15", + resolution="5min", + by_region=True, + ) + ``` """ start_ts, end_ts = parse_date_range(start, end) - if by_region: - if self._needs_historical(start_ts, "forecast"): - df = self._get_archive().fetch_historical( - endpoint="/np4-745-cd/spp_hrly_actual_fcast_geo", - start=start_ts, - end=end_ts, - ) + # Validate resolution + resolution = resolution.lower() + if resolution not in ("hourly", "5min", "5-min", "5_min"): + raise ValueError( + f"Invalid resolution: {resolution}. Use 'hourly' or '5min'." + ) + + use_5min = resolution in ("5min", "5-min", "5_min") + + if use_5min: + # 5-minute data endpoints + if by_region: + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-746-cd/spp_actual_5min_avg_values_geo", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_spp_actual_5min_avg_values_geo( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - df = self.get_spp_hourly_actual_forecast_geo( - posted_datetime_from=format_api_date(start_ts), - posted_datetime_to=format_api_date(end_ts), - ) + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-738-cd/spp_actual_5min_avg_values", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_spp_actual_5min_avg_values( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - if self._needs_historical(start_ts, "forecast"): - df = self._get_archive().fetch_historical( - endpoint="/np4-737-cd/spp_hrly_avrg_actl_fcast", - start=start_ts, - end=end_ts, - ) + # Hourly data endpoints (default) + if by_region: + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-745-cd/spp_hrly_actual_fcast_geo", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_spp_hourly_actual_forecast_geo( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) else: - df = self.get_spp_hourly_average_actual_forecast( - posted_datetime_from=format_api_date(start_ts), - posted_datetime_to=format_api_date(end_ts), - ) + if self._needs_historical(start_ts, "forecast"): + df = self._get_archive().fetch_historical( + endpoint="/np4-737-cd/spp_hrly_avrg_actl_fcast", + start=start_ts, + end=end_ts, + ) + else: + df = self.get_spp_hourly_average_actual_forecast( + posted_datetime_from=format_api_date(start_ts), + posted_datetime_to=format_api_date(end_ts), + ) df = filter_by_date(df, start_ts, end_ts, date_column="Posted Datetime") return standardize_columns(df) + # ============================================================================ + # System-Wide Generation and Transmission Data + # ============================================================================ + + def get_dc_tie_flows( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get DC Tie flow data from state estimator. + + DC Ties connect ERCOT to neighboring grids (Eastern Interconnection + and Mexico). This endpoint provides the scheduled and actual flows + across these ties. + + EMIL ID: NP6-626-CD + + Args: + start: Start date - "today", "yesterday", or ISO format + end: End date (defaults to start + 1 day) + + Returns: + DataFrame with DC tie flow data + + Note: + This data is only available through the archive API for historical + dates. Real-time access may require pyercot updates. + + Example: + ```python + ercot = ERCOT(auth=auth) + dc_ties = ercot.get_dc_tie_flows(start="2024-01-15") + ``` + """ + start_ts, end_ts = parse_date_range(start, end) + + # DC tie data via archive API + df = self._get_archive().fetch_historical( + endpoint="/np6-626-cd/dc_tie", + start=start_ts, + end=end_ts, + ) + + df = filter_by_date(df, start_ts, end_ts) + return standardize_columns(df) + + def get_total_generation( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get total ERCOT system generation. + + Provides the total MW generation across the ERCOT system from + the state estimator. + + EMIL ID: NP6-625-CD + + Args: + start: Start date - "today", "yesterday", or ISO format + end: End date (defaults to start + 1 day) + + Returns: + DataFrame with total system generation data + + Note: + This data is only available through the archive API for historical + dates. Real-time access may require pyercot updates. + + Example: + ```python + ercot = ERCOT(auth=auth) + total_gen = ercot.get_total_generation(start="2024-01-15") + ``` + """ + start_ts, end_ts = parse_date_range(start, end) + + # Total generation via archive API + df = self._get_archive().fetch_historical( + endpoint="/np6-625-cd/se_totalgen", + start=start_ts, + end=end_ts, + ) + + df = filter_by_date(df, start_ts, end_ts) + return standardize_columns(df) + + def get_system_wide_actuals( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get system-wide actual values per SCED interval. + + Provides actual system-wide metrics from each SCED execution + including load, generation, and reserves. + + EMIL ID: NP6-235-CD + + Args: + start: Start date - "today", "yesterday", or ISO format + end: End date (defaults to start + 1 day) + + Returns: + DataFrame with system-wide actual data + + Note: + This data is only available through the archive API for historical + dates. Real-time access may require pyercot updates. + + Example: + ```python + ercot = ERCOT(auth=auth) + actuals = ercot.get_system_wide_actuals(start="2024-01-15") + ``` + """ + start_ts, end_ts = parse_date_range(start, end) + + # System-wide actuals via archive API + df = self._get_archive().fetch_historical( + endpoint="/np6-235-cd/sys_wide_actuals", + start=start_ts, + end=end_ts, + ) + + df = filter_by_date(df, start_ts, end_ts) + return standardize_columns(df) + # ============================================================================ # 60-Day Disclosure Reports # ============================================================================ diff --git a/tinygrid/ercot/client.py b/tinygrid/ercot/client.py index 2e58b63..f730682 100644 --- a/tinygrid/ercot/client.py +++ b/tinygrid/ercot/client.py @@ -36,6 +36,7 @@ GridRetryExhaustedError, GridTimeoutError, ) +from ..utils.rate_limiter import ERCOT_REQUESTS_PER_MINUTE, RateLimiter if TYPE_CHECKING: from .archive import ERCOTArchive @@ -77,6 +78,8 @@ class ERCOTBase(BaseISOClient): retry_max_wait: Maximum wait time between retries in seconds. Defaults to 60.0. page_size: Number of records per page when fetching data. Defaults to 10000. max_concurrent_requests: Maximum number of concurrent page requests. Defaults to 5. + rate_limit_enabled: Whether to enforce rate limiting. Defaults to True. + requests_per_minute: Maximum requests per minute. Defaults to 30 (ERCOT limit). """ base_url: str = field(default="https://api.ercot.com/api/public-reports") @@ -94,6 +97,10 @@ class ERCOTBase(BaseISOClient): page_size: int = field(default=10000, kw_only=True) max_concurrent_requests: int = field(default=5, kw_only=True) + # Rate limiting configuration + rate_limit_enabled: bool = field(default=True, kw_only=True) + requests_per_minute: float = field(default=ERCOT_REQUESTS_PER_MINUTE, kw_only=True) + _client: ERCOTClient | AuthenticatedClient | None = field( default=None, init=False, repr=False ) @@ -101,6 +108,7 @@ class ERCOTBase(BaseISOClient): default=None, init=False, repr=False ) _archive: Any = field(default=None, init=False, repr=False) + _rate_limiter: RateLimiter | None = field(default=None, init=False, repr=False) @property def iso_name(self) -> str: @@ -166,6 +174,21 @@ def _get_client(self) -> ERCOTClient | AuthenticatedClient: return self._client + def _get_rate_limiter(self) -> RateLimiter | None: + """Get or create the rate limiter. + + Returns: + RateLimiter instance if rate limiting is enabled, None otherwise + """ + if not self.rate_limit_enabled: + return None + + if self._rate_limiter is None: + self._rate_limiter = RateLimiter( + requests_per_minute=self.requests_per_minute + ) + return self._rate_limiter + def __enter__(self) -> ERCOTBase: """Enter a context manager for the client.""" self._entered_client = self._get_client() @@ -472,6 +495,11 @@ def _call_endpoint_raw( Dictionary containing the response data """ try: + # Apply rate limiting before making the request + rate_limiter = self._get_rate_limiter() + if rate_limiter is not None: + rate_limiter.acquire() + client = self._get_client() response = endpoint_module.sync(client=client, **kwargs) diff --git a/tinygrid/ercot/dashboard.py b/tinygrid/ercot/dashboard.py index 441f675..3388cfa 100644 --- a/tinygrid/ercot/dashboard.py +++ b/tinygrid/ercot/dashboard.py @@ -1,27 +1,44 @@ """Dashboard/JSON methods for ERCOT data access. -NOTE: ERCOT's public dashboard data is not available via documented JSON endpoints. -The methods in this module are placeholders that return empty data or default values. - -For real-time grid data, use the authenticated API methods instead: -- System load: get_actual_system_load_by_weather_zone() -- Generation: get_generation_by_resource_type() -- Forecasts: get_load_forecast_by_weather_zone(), get_wpp_hourly_average_actual_forecast() - -These dashboard methods may be implemented in the future if ERCOT provides public -JSON endpoints, or by scraping the ERCOT dashboard website. +This module provides access to ERCOT's public dashboard data via undocumented +JSON endpoints. These endpoints do NOT require authentication and provide +real-time grid status information. + +Endpoints used: +- https://www.ercot.com/api/1/services/read/dashboards/todays-outlook.json +- https://www.ercot.com/api/1/services/read/dashboards/daily-prc.json +- https://www.ercot.com/api/1/services/read/dashboards/combinedWindSolar.json +- https://www.ercot.com/api/1/services/read/dashboards/supplyDemand.json + +Note: These are undocumented endpoints that power ERCOT's public dashboard. +They may change without notice. """ from __future__ import annotations import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum +from typing import Any +import httpx import pandas as pd +from ..constants.ercot import ERCOT_TIMEZONE + logger = logging.getLogger(__name__) +# Dashboard JSON endpoints (undocumented but functional) +DASHBOARD_BASE_URL = "https://www.ercot.com/api/1/services/read/dashboards" +TODAYS_OUTLOOK_URL = f"{DASHBOARD_BASE_URL}/todays-outlook.json" +DAILY_PRC_URL = f"{DASHBOARD_BASE_URL}/daily-prc.json" +COMBINED_WIND_SOLAR_URL = f"{DASHBOARD_BASE_URL}/combinedWindSolar.json" +SUPPLY_DEMAND_URL = f"{DASHBOARD_BASE_URL}/supplyDemand.json" +FUEL_MIX_URL = f"{DASHBOARD_BASE_URL}/fuel-mix.json" + +# Default timeout for dashboard requests +DASHBOARD_TIMEOUT = 15.0 + class GridCondition(str, Enum): """ERCOT grid operating conditions.""" @@ -29,21 +46,58 @@ class GridCondition(str, Enum): NORMAL = "normal" CONSERVATION = "conservation" WATCH = "watch" + ADVISORY = "advisory" EMERGENCY = "emergency" + EEA1 = "eea1" + EEA2 = "eea2" + EEA3 = "eea3" UNKNOWN = "unknown" + @classmethod + def from_string(cls, value: str | None) -> GridCondition: + """Parse a condition string from the API.""" + if not value: + return cls.UNKNOWN + value_lower = value.lower().strip() + # Map common variations + mappings = { + "normal": cls.NORMAL, + "normal operations": cls.NORMAL, + "conservation": cls.CONSERVATION, + "conservation appeal": cls.CONSERVATION, + "watch": cls.WATCH, + "weather watch": cls.WATCH, + "advisory": cls.ADVISORY, + "operating condition notice": cls.ADVISORY, + "emergency": cls.EMERGENCY, + "eea1": cls.EEA1, + "eea 1": cls.EEA1, + "energy emergency alert 1": cls.EEA1, + "eea2": cls.EEA2, + "eea 2": cls.EEA2, + "energy emergency alert 2": cls.EEA2, + "eea3": cls.EEA3, + "eea 3": cls.EEA3, + "energy emergency alert 3": cls.EEA3, + } + return mappings.get(value_lower, cls.UNKNOWN) + @dataclass class GridStatus: """Current grid operating status.""" condition: GridCondition - current_frequency: float current_load: float capacity: float reserves: float timestamp: pd.Timestamp + current_frequency: float = 60.0 message: str = "" + peak_forecast: float = 0.0 + wind_output: float = 0.0 + solar_output: float = 0.0 + prc: float = 0.0 # Physical Responsive Capability @classmethod def unavailable(cls) -> GridStatus: @@ -54,131 +108,610 @@ def unavailable(cls) -> GridStatus: current_load=0.0, capacity=0.0, reserves=0.0, - timestamp=pd.Timestamp.now(tz="US/Central"), - message="Dashboard data not available - use authenticated API methods instead", + timestamp=pd.Timestamp.now(tz=ERCOT_TIMEZONE), + message="Dashboard data not available", ) +@dataclass +class FuelMixEntry: + """A single fuel type entry in the fuel mix.""" + + fuel_type: str + generation_mw: float + percentage: float + timestamp: pd.Timestamp + + +@dataclass +class RenewableStatus: + """Current renewable generation status.""" + + wind_mw: float + solar_mw: float + wind_forecast_mw: float + solar_forecast_mw: float + wind_capacity_mw: float + solar_capacity_mw: float + timestamp: pd.Timestamp + additional_data: dict[str, Any] = field(default_factory=dict) + + +def _fetch_json(url: str, timeout: float = DASHBOARD_TIMEOUT) -> dict[str, Any] | None: + """Fetch JSON data from a dashboard endpoint. + + Args: + url: The endpoint URL + timeout: Request timeout in seconds + + Returns: + Parsed JSON dict or None if request fails + """ + try: + with httpx.Client(timeout=timeout) as client: + response = client.get(url) + response.raise_for_status() + return response.json() + except httpx.TimeoutException: + logger.warning(f"Dashboard request timed out: {url}") + return None + except httpx.HTTPStatusError as e: + logger.warning( + f"Dashboard request failed with status {e.response.status_code}: {url}" + ) + return None + except Exception as e: + logger.warning(f"Dashboard request failed: {url} - {e}") + return None + + +def _safe_float(value: Any, default: float = 0.0) -> float: + """Safely convert a value to float.""" + if value is None: + return default + try: + return float(value) + except (ValueError, TypeError): + return default + + +def _parse_timestamp(value: Any) -> pd.Timestamp: + """Parse a timestamp from the API response.""" + now: pd.Timestamp = pd.Timestamp.now(tz=ERCOT_TIMEZONE) + if value is None: + return now + try: + # Try parsing as epoch milliseconds first + if isinstance(value, (int, float)) and value > 1_000_000_000_000: + result = pd.Timestamp(value, unit="ms", tz=ERCOT_TIMEZONE) + elif isinstance(value, (int, float)): + result = pd.Timestamp(value, unit="s", tz=ERCOT_TIMEZONE) + else: + result = pd.Timestamp(value, tz=ERCOT_TIMEZONE) + # Handle NaT case - result is pd.NaT doesn't work with pyright + # so we use the hash comparison trick + if result is pd.NaT or str(result) == "NaT": + return now + # Cast to Timestamp to satisfy type checker + return pd.Timestamp(result) + except Exception: + return now + + class ERCOTDashboardMixin: """Mixin class providing dashboard/JSON methods. - NOTE: These methods are placeholders. ERCOT does not provide documented - public JSON endpoints for dashboard data. Use authenticated API methods - for real data: + These methods access ERCOT's public dashboard JSON endpoints which do NOT + require authentication. They provide real-time grid status information + that powers the ERCOT dashboard at ercot.com. - - System load: get_actual_system_load_by_weather_zone() - - Forecasts: get_load_forecast_by_weather_zone() - - Wind/Solar: get_wpp_hourly_average_actual_forecast(), get_spp_hourly_average_actual_forecast() + Note: These endpoints are undocumented and may change without notice. + For production use with SLA requirements, consider using the authenticated + API endpoints instead. """ def get_status(self) -> GridStatus: - """Get current grid operating status. + """Get current grid operating status from ERCOT dashboard. - NOTE: This method returns placeholder data. ERCOT does not provide - a public JSON API for grid status. For real data, use: - - get_actual_system_load_by_weather_zone() for current load - - Check ercot.com dashboard for grid conditions + Fetches real-time grid status including: + - Operating condition (normal, conservation, emergency, EEA levels) + - Current system load + - Available capacity + - Operating reserves + - Wind and solar output Returns: - GridStatus object (placeholder with unavailable message) + GridStatus object with current grid conditions + + Example: + ```python + ercot = ERCOT() + status = ercot.get_status() + print(f"Condition: {status.condition}") + print(f"Current Load: {status.current_load:,.0f} MW") + print(f"Reserves: {status.reserves:,.0f} MW") + ``` """ - logger.warning( - "get_status() returns placeholder data - " - "ERCOT does not provide public JSON endpoints for dashboard data" - ) - return GridStatus.unavailable() - - def get_fuel_mix(self, date: str = "today") -> pd.DataFrame: - """Get generation fuel mix data. - - NOTE: This method returns empty DataFrame. ERCOT does not provide - a public JSON API for fuel mix. For real data, use: - - get_generation_by_resource_type() (requires auth) - - Check ercot.com fuel mix dashboard + # Fetch today's outlook data + data = _fetch_json(TODAYS_OUTLOOK_URL) + if not data: + logger.warning("Failed to fetch grid status from dashboard") + return GridStatus.unavailable() + + try: + # Parse the response - structure varies by time of day + current = data.get("current", data) + + # Extract values with safe defaults + condition_str = current.get("condition") or current.get("status") or "" + condition = GridCondition.from_string(condition_str) + + # Get load and capacity values + current_load = _safe_float(current.get("demand") or current.get("load")) + capacity = _safe_float( + current.get("capacity") or current.get("totalCapacity") + ) + reserves = _safe_float( + current.get("reserves") or current.get("operatingReserves") + ) + + # Calculate reserves if not directly available + if reserves == 0.0 and capacity > 0 and current_load > 0: + reserves = capacity - current_load + + # Get renewable data if available + wind = _safe_float(current.get("windOutput") or current.get("wind")) + solar = _safe_float(current.get("solarOutput") or current.get("solar")) + + # Get peak forecast + peak = _safe_float(current.get("peakForecast") or current.get("peak")) + + # Get timestamp + ts = _parse_timestamp( + current.get("lastUpdated") or current.get("timestamp") + ) + + # Get PRC (Physical Responsive Capability) if available + prc = _safe_float(current.get("prc") or current.get("physicalResponsive")) + + # Build message from any alerts + message = current.get("message") or current.get("alert") or "" + if condition != GridCondition.NORMAL and not message: + message = f"Grid operating in {condition.value} condition" + + return GridStatus( + condition=condition, + current_load=current_load, + capacity=capacity, + reserves=reserves, + timestamp=ts, + peak_forecast=peak, + wind_output=wind, + solar_output=solar, + prc=prc, + message=message, + ) + + except Exception as e: + logger.warning(f"Failed to parse grid status: {e}") + return GridStatus.unavailable() + + def get_fuel_mix( + self, as_dataframe: bool = True + ) -> pd.DataFrame | list[FuelMixEntry]: + """Get current generation fuel mix data. + + Fetches the current generation breakdown by fuel type from the + ERCOT dashboard. This shows real-time MW output by fuel source. Args: - date: Date to fetch ("today", "yesterday", or YYYY-MM-DD) + as_dataframe: If True, return results as DataFrame. If False, + return list of FuelMixEntry objects. Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame or list with fuel mix data including: + - fuel_type: Type of fuel (gas, coal, nuclear, wind, solar, etc.) + - generation_mw: Current generation in MW + - percentage: Percentage of total generation + + Example: + ```python + ercot = ERCOT() + fuel_mix = ercot.get_fuel_mix() + print(fuel_mix) + # fuel_type generation_mw percentage + # 0 gas 25000.0 45.2 + # 1 wind 18000.0 32.5 + # 2 solar 8000.0 14.5 + # ... + ``` """ - logger.warning( - "get_fuel_mix() returns empty data - " - "ERCOT does not provide public JSON endpoints for fuel mix. " - "Use get_generation_by_resource_type() with authentication instead." - ) - return pd.DataFrame() + data = _fetch_json(FUEL_MIX_URL) + if not data: + logger.warning("Failed to fetch fuel mix from dashboard") + if as_dataframe: + return pd.DataFrame( + columns=["fuel_type", "generation_mw", "percentage", "timestamp"] + ) + return [] + + try: + entries: list[FuelMixEntry] = [] + ts = _parse_timestamp(data.get("lastUpdated") or data.get("timestamp")) + + # Parse fuel mix entries - structure may be list or nested + fuel_data = data.get("data") or data.get("fuelMix") or data + if isinstance(fuel_data, list): + total_gen = sum( + _safe_float(f.get("gen") or f.get("generation") or f.get("mw")) + for f in fuel_data + ) + + for item in fuel_data: + fuel_type = ( + item.get("fuel") + or item.get("fuelType") + or item.get("type") + or "unknown" + ) + gen_mw = _safe_float( + item.get("gen") or item.get("generation") or item.get("mw") + ) + pct = _safe_float(item.get("percent") or item.get("percentage")) + if pct == 0.0 and total_gen > 0: + pct = (gen_mw / total_gen) * 100 + + entries.append( + FuelMixEntry( + fuel_type=fuel_type, + generation_mw=gen_mw, + percentage=pct, + timestamp=ts, + ) + ) + + if as_dataframe: + if not entries: + return pd.DataFrame( + columns=[ + "fuel_type", + "generation_mw", + "percentage", + "timestamp", + ] + ) + return pd.DataFrame( + [ + { + "fuel_type": e.fuel_type, + "generation_mw": e.generation_mw, + "percentage": e.percentage, + "timestamp": e.timestamp, + } + for e in entries + ] + ) + return entries + + except Exception as e: + logger.warning(f"Failed to parse fuel mix: {e}") + if as_dataframe: + return pd.DataFrame( + columns=["fuel_type", "generation_mw", "percentage", "timestamp"] + ) + return [] + + def get_renewable_generation(self) -> RenewableStatus: + """Get current renewable generation data (wind and solar). + + Fetches combined wind and solar generation data including current + output, forecasts, and installed capacity. - def get_energy_storage_resources(self) -> pd.DataFrame: - """Get energy storage resource (ESR) data. + Returns: + RenewableStatus object with current renewable data + + Example: + ```python + ercot = ERCOT() + renewable = ercot.get_renewable_generation() + print(f"Wind: {renewable.wind_mw:,.0f} MW") + print(f"Solar: {renewable.solar_mw:,.0f} MW") + print(f"Total Renewable: {renewable.wind_mw + renewable.solar_mw:,.0f} MW") + ``` + """ + data = _fetch_json(COMBINED_WIND_SOLAR_URL) + if not data: + logger.warning("Failed to fetch renewable generation from dashboard") + return RenewableStatus( + wind_mw=0.0, + solar_mw=0.0, + wind_forecast_mw=0.0, + solar_forecast_mw=0.0, + wind_capacity_mw=0.0, + solar_capacity_mw=0.0, + timestamp=pd.Timestamp.now(tz=ERCOT_TIMEZONE), + ) + + try: + current = data.get("current", data) + ts = _parse_timestamp(current.get("lastUpdated") or data.get("lastUpdated")) + + return RenewableStatus( + wind_mw=_safe_float(current.get("windActual") or current.get("wind")), + solar_mw=_safe_float( + current.get("solarActual") or current.get("solar") + ), + wind_forecast_mw=_safe_float( + current.get("windForecast") or current.get("windFcst") + ), + solar_forecast_mw=_safe_float( + current.get("solarForecast") or current.get("solarFcst") + ), + wind_capacity_mw=_safe_float( + current.get("windCapacity") or current.get("windCap") + ), + solar_capacity_mw=_safe_float( + current.get("solarCapacity") or current.get("solarCap") + ), + timestamp=ts, + additional_data=current, + ) + + except Exception as e: + logger.warning(f"Failed to parse renewable generation: {e}") + return RenewableStatus( + wind_mw=0.0, + solar_mw=0.0, + wind_forecast_mw=0.0, + solar_forecast_mw=0.0, + wind_capacity_mw=0.0, + solar_capacity_mw=0.0, + timestamp=pd.Timestamp.now(tz=ERCOT_TIMEZONE), + ) + + def get_supply_demand(self) -> pd.DataFrame: + """Get supply and demand curve data. + + Fetches the current supply/demand balance including hourly forecasts + and capacity projections. - NOTE: This method returns empty DataFrame. For ESR data, use - authenticated API methods. + Returns: + DataFrame with supply and demand data by hour + + Example: + ```python + ercot = ERCOT() + supply_demand = ercot.get_supply_demand() + print(supply_demand.columns) + # ['hour', 'demand', 'supply', 'reserves', 'timestamp'] + ``` + """ + data = _fetch_json(SUPPLY_DEMAND_URL) + if not data: + logger.warning("Failed to fetch supply/demand from dashboard") + return pd.DataFrame( + columns=["hour", "demand", "supply", "reserves", "timestamp"] + ) + + try: + records = [] + ts = _parse_timestamp(data.get("lastUpdated")) + + hourly_data = data.get("data") or data.get("hourly") or [] + for item in hourly_data: + records.append( + { + "hour": item.get("hour") or item.get("hourEnding"), + "demand": _safe_float(item.get("demand") or item.get("load")), + "supply": _safe_float( + item.get("supply") or item.get("capacity") + ), + "reserves": _safe_float(item.get("reserves")), + "timestamp": ts, + } + ) + + if not records: + return pd.DataFrame( + columns=["hour", "demand", "supply", "reserves", "timestamp"] + ) + + return pd.DataFrame(records) + + except Exception as e: + logger.warning(f"Failed to parse supply/demand: {e}") + return pd.DataFrame( + columns=["hour", "demand", "supply", "reserves", "timestamp"] + ) + + def get_daily_prices(self) -> pd.DataFrame: + """Get daily price summary from dashboard. + + Fetches the daily price summary including peak and average prices + from the ERCOT dashboard. Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame with daily price data + + Example: + ```python + ercot = ERCOT() + prices = ercot.get_daily_prices() + ``` """ - logger.warning( - "get_energy_storage_resources() returns empty data - " - "use authenticated API methods for ESR data" - ) - return pd.DataFrame() + data = _fetch_json(DAILY_PRC_URL) + if not data: + logger.warning("Failed to fetch daily prices from dashboard") + return pd.DataFrame() + + try: + records = [] + ts = _parse_timestamp(data.get("lastUpdated")) + + price_data = data.get("data") or data.get("prices") or data + if isinstance(price_data, list): + for item in price_data: + records.append( + { + "settlement_point": item.get("settlementPoint") + or item.get("sp"), + "price": _safe_float(item.get("price") or item.get("spp")), + "peak_price": _safe_float(item.get("peakPrice")), + "avg_price": _safe_float(item.get("avgPrice")), + "timestamp": ts, + } + ) + + if not records: + return pd.DataFrame() + + return pd.DataFrame(records) + + except Exception as e: + logger.warning(f"Failed to parse daily prices: {e}") + return pd.DataFrame() def get_system_wide_demand(self) -> pd.DataFrame: - """Get system-wide demand data. + """Get system-wide demand data from dashboard. - NOTE: This method returns empty DataFrame. For demand data, use: - - get_actual_system_load_by_weather_zone() (current load) - - get_load_forecast_by_weather_zone() (forecasts) + Fetches current and forecasted system-wide demand from the + ERCOT dashboard. Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame with system-wide demand data """ - logger.warning( - "get_system_wide_demand() returns empty data - " - "use get_actual_system_load_by_weather_zone() instead" - ) - return pd.DataFrame() + data = _fetch_json(TODAYS_OUTLOOK_URL) + if not data: + logger.warning("Failed to fetch system-wide demand from dashboard") + return pd.DataFrame() + + try: + records = [] + ts = _parse_timestamp(data.get("lastUpdated")) + + # Get current and forecasted demand + current = data.get("current", {}) + hourly = data.get("hourly") or data.get("data") or [] + + # Add current demand + if current: + records.append( + { + "hour": "current", + "demand": _safe_float( + current.get("demand") or current.get("load") + ), + "capacity": _safe_float(current.get("capacity")), + "reserves": _safe_float(current.get("reserves")), + "timestamp": ts, + } + ) + + # Add hourly forecasts + for item in hourly: + records.append( + { + "hour": item.get("hour") or item.get("hourEnding"), + "demand": _safe_float(item.get("demand") or item.get("load")), + "capacity": _safe_float(item.get("capacity")), + "reserves": _safe_float(item.get("reserves")), + "timestamp": ts, + } + ) + + if not records: + return pd.DataFrame() + + return pd.DataFrame(records) + + except Exception as e: + logger.warning(f"Failed to parse system-wide demand: {e}") + return pd.DataFrame() - def get_renewable_generation(self) -> pd.DataFrame: - """Get renewable generation data (wind and solar). + def get_energy_storage_resources(self) -> pd.DataFrame: + """Get energy storage resource (ESR) data. - NOTE: This method returns empty DataFrame. For renewable data, use: - - get_wpp_hourly_average_actual_forecast() (wind) - - get_spp_hourly_average_actual_forecast() (solar) + Note: ESR data may not be available via dashboard endpoints. + Use authenticated API methods for detailed ESR data. Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame with ESR data if available, empty DataFrame otherwise """ - logger.warning( - "get_renewable_generation() returns empty data - " - "use get_wpp_hourly_average_actual_forecast() or " - "get_spp_hourly_average_actual_forecast() instead" - ) - return pd.DataFrame() + # ESR data is typically not in the public dashboard + # Try to get it from the supply/demand or outlook data + data = _fetch_json(TODAYS_OUTLOOK_URL) + if not data: + return pd.DataFrame() + + try: + current = data.get("current", {}) + esr = current.get("esr") or current.get("storage") or current.get("battery") + + if esr is None: + return pd.DataFrame() + + if isinstance(esr, dict): + return pd.DataFrame( + [ + { + "charging_mw": _safe_float(esr.get("charging")), + "discharging_mw": _safe_float(esr.get("discharging")), + "net_mw": _safe_float(esr.get("net")), + "capacity_mw": _safe_float(esr.get("capacity")), + "timestamp": _parse_timestamp(data.get("lastUpdated")), + } + ] + ) + + return pd.DataFrame() + + except Exception as e: + logger.warning(f"Failed to parse ESR data: {e}") + return pd.DataFrame() def get_capacity_committed(self) -> pd.DataFrame: """Get committed generation capacity data. - NOTE: This method returns empty DataFrame. - Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame with committed capacity data if available """ - logger.warning( - "get_capacity_committed() returns empty data - endpoint not available" - ) - return pd.DataFrame() + data = _fetch_json(SUPPLY_DEMAND_URL) + if not data: + return pd.DataFrame() + + try: + records = [] + ts = _parse_timestamp(data.get("lastUpdated")) + + hourly = data.get("data") or data.get("hourly") or [] + for item in hourly: + records.append( + { + "hour": item.get("hour") or item.get("hourEnding"), + "committed_capacity": _safe_float( + item.get("committed") or item.get("supply") + ), + "available_capacity": _safe_float( + item.get("available") or item.get("capacity") + ), + "timestamp": ts, + } + ) + + if not records: + return pd.DataFrame() + + return pd.DataFrame(records) + + except Exception as e: + logger.warning(f"Failed to parse committed capacity: {e}") + return pd.DataFrame() def get_capacity_forecast(self) -> pd.DataFrame: """Get capacity forecast data. - NOTE: This method returns empty DataFrame. - Returns: - Empty DataFrame (placeholder - endpoint not available) + DataFrame with capacity forecast data if available """ - logger.warning( - "get_capacity_forecast() returns empty data - endpoint not available" - ) - return pd.DataFrame() + # Use the same data as supply/demand + return self.get_supply_demand() diff --git a/tinygrid/ercot/eia.py b/tinygrid/ercot/eia.py new file mode 100644 index 0000000..170c80e --- /dev/null +++ b/tinygrid/ercot/eia.py @@ -0,0 +1,368 @@ +"""EIA (Energy Information Administration) API integration for ERCOT data. + +The EIA provides supplementary data for ERCOT including: +- Hourly demand and generation (from 2019) +- Historical capacity and fuel mix data +- Retail electricity sales + +This module provides access to EIA data as an alternative or supplement +to ERCOT's native API, especially useful for: +- Data before December 2023 (when ERCOT's API launched) +- Cross-validation of ERCOT data +- Additional metrics not in ERCOT's API + +API Documentation: https://www.eia.gov/opendata/documentation.php +""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx +import pandas as pd + +from ..constants.ercot import ERCOT_TIMEZONE + +logger = logging.getLogger(__name__) + +# EIA API endpoints +EIA_API_BASE_URL = "https://api.eia.gov/v2" + +# ERCOT is identified as "ERCO" balancing authority in EIA data +ERCOT_BA_CODE = "ERCO" + +# EIA bulk download URL for Electric Balancing Authority data +EIA_BULK_DOWNLOAD_URL = "https://www.eia.gov/opendata/bulk/EBA.zip" + + +class EIAClient: + """Client for accessing ERCOT data via the EIA API. + + The EIA (Energy Information Administration) provides free access to + US energy data including hourly electricity demand and generation + by balancing authority. + + ERCOT data is available under the balancing authority code "ERCO". + + Note: Requires a free API key from https://www.eia.gov/opendata/register.php + + Args: + api_key: EIA API key (required for most endpoints) + timeout: Request timeout in seconds + + Example: + ```python + from tinygrid.ercot.eia import EIAClient + + eia = EIAClient(api_key="your-api-key") + + # Get hourly demand for ERCOT + demand = eia.get_demand(start="2024-01-01", end="2024-01-07") + + # Get generation by fuel type + gen = eia.get_generation_by_fuel(start="2024-01-01", end="2024-01-07") + ``` + """ + + def __init__( + self, + api_key: str | None = None, + timeout: float = 30.0, + ) -> None: + """Initialize the EIA client. + + Args: + api_key: EIA API key. Get one at https://www.eia.gov/opendata/register.php + timeout: Request timeout in seconds + """ + self.api_key = api_key + self.timeout = timeout + self._base_url = EIA_API_BASE_URL + + def _make_request( + self, + endpoint: str, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Make a request to the EIA API. + + Args: + endpoint: API endpoint path + params: Query parameters + + Returns: + Parsed JSON response + + Raises: + ValueError: If API key is required but not provided + httpx.HTTPError: If request fails + """ + if self.api_key is None: + raise ValueError( + "EIA API key required. Get one at https://www.eia.gov/opendata/register.php" + ) + + url = f"{self._base_url}/{endpoint}" + request_params = {"api_key": self.api_key} + if params: + request_params.update(params) + + try: + with httpx.Client(timeout=self.timeout) as client: + response = client.get(url, params=request_params) + response.raise_for_status() + return response.json() + except httpx.TimeoutException: + logger.error(f"EIA API request timed out: {url}") + raise + except httpx.HTTPStatusError as e: + logger.error(f"EIA API request failed: {e.response.status_code} - {url}") + raise + + def get_demand( + self, + start: str | pd.Timestamp, + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get hourly demand data for ERCOT. + + Fetches hourly electricity demand (load) for the ERCOT balancing + authority from the EIA API. + + Args: + start: Start date (YYYY-MM-DD format or Timestamp) + end: End date (defaults to start + 7 days) + + Returns: + DataFrame with columns: timestamp, demand_mw + + Example: + ```python + eia = EIAClient(api_key="your-key") + demand = eia.get_demand(start="2024-01-01", end="2024-01-07") + ``` + """ + start_ts = pd.Timestamp(start) + end_ts = pd.Timestamp(end) if end else start_ts + pd.Timedelta(days=7) + + params = { + "frequency": "hourly", + "data[0]": "value", + "facets[respondent][]": ERCOT_BA_CODE, + "facets[type][]": "D", # Demand + "start": start_ts.strftime("%Y-%m-%dT00"), + "end": end_ts.strftime("%Y-%m-%dT23"), + "sort[0][column]": "period", + "sort[0][direction]": "asc", + } + + try: + response = self._make_request("electricity/rto/region-data/data", params) + data = response.get("response", {}).get("data", []) + + if not data: + return pd.DataFrame(columns=["timestamp", "demand_mw"]) + + records = [] + for item in data: + records.append( + { + "timestamp": pd.Timestamp( + item.get("period"), tz=ERCOT_TIMEZONE + ), + "demand_mw": float(item.get("value", 0)), + } + ) + + return pd.DataFrame(records) + + except Exception as e: + logger.error(f"Failed to fetch EIA demand data: {e}") + return pd.DataFrame(columns=["timestamp", "demand_mw"]) + + def get_generation( + self, + start: str | pd.Timestamp, + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get hourly net generation data for ERCOT. + + Fetches hourly electricity generation for the ERCOT balancing + authority from the EIA API. + + Args: + start: Start date + end: End date (defaults to start + 7 days) + + Returns: + DataFrame with columns: timestamp, generation_mw + """ + start_ts = pd.Timestamp(start) + end_ts = pd.Timestamp(end) if end else start_ts + pd.Timedelta(days=7) + + params = { + "frequency": "hourly", + "data[0]": "value", + "facets[respondent][]": ERCOT_BA_CODE, + "facets[type][]": "NG", # Net Generation + "start": start_ts.strftime("%Y-%m-%dT00"), + "end": end_ts.strftime("%Y-%m-%dT23"), + "sort[0][column]": "period", + "sort[0][direction]": "asc", + } + + try: + response = self._make_request("electricity/rto/region-data/data", params) + data = response.get("response", {}).get("data", []) + + if not data: + return pd.DataFrame(columns=["timestamp", "generation_mw"]) + + records = [] + for item in data: + records.append( + { + "timestamp": pd.Timestamp( + item.get("period"), tz=ERCOT_TIMEZONE + ), + "generation_mw": float(item.get("value", 0)), + } + ) + + return pd.DataFrame(records) + + except Exception as e: + logger.error(f"Failed to fetch EIA generation data: {e}") + return pd.DataFrame(columns=["timestamp", "generation_mw"]) + + def get_generation_by_fuel( + self, + start: str | pd.Timestamp, + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get hourly generation by fuel type for ERCOT. + + Fetches hourly electricity generation broken down by fuel source + (coal, natural gas, nuclear, wind, solar, etc.). + + Args: + start: Start date + end: End date (defaults to start + 7 days) + + Returns: + DataFrame with columns: timestamp, fuel_type, generation_mw + """ + start_ts = pd.Timestamp(start) + end_ts = pd.Timestamp(end) if end else start_ts + pd.Timedelta(days=7) + + params = { + "frequency": "hourly", + "data[0]": "value", + "facets[respondent][]": ERCOT_BA_CODE, + "start": start_ts.strftime("%Y-%m-%dT00"), + "end": end_ts.strftime("%Y-%m-%dT23"), + "sort[0][column]": "period", + "sort[0][direction]": "asc", + } + + try: + response = self._make_request("electricity/rto/fuel-type-data/data", params) + data = response.get("response", {}).get("data", []) + + if not data: + return pd.DataFrame(columns=["timestamp", "fuel_type", "generation_mw"]) + + records = [] + for item in data: + fuel_type = item.get("fueltype", "unknown") + # Map EIA fuel type codes to readable names + fuel_name = _map_fuel_type(fuel_type) + + records.append( + { + "timestamp": pd.Timestamp( + item.get("period"), tz=ERCOT_TIMEZONE + ), + "fuel_type": fuel_name, + "generation_mw": float(item.get("value", 0)), + } + ) + + return pd.DataFrame(records) + + except Exception as e: + logger.error(f"Failed to fetch EIA generation by fuel: {e}") + return pd.DataFrame(columns=["timestamp", "fuel_type", "generation_mw"]) + + def get_interchange( + self, + start: str | pd.Timestamp, + end: str | pd.Timestamp | None = None, + ) -> pd.DataFrame: + """Get hourly interchange data for ERCOT. + + Fetches net interchange (imports minus exports) with neighboring + regions. Note: ERCOT has limited interconnections due to Texas's + isolated grid. + + Args: + start: Start date + end: End date (defaults to start + 7 days) + + Returns: + DataFrame with columns: timestamp, interchange_mw + """ + start_ts = pd.Timestamp(start) + end_ts = pd.Timestamp(end) if end else start_ts + pd.Timedelta(days=7) + + params = { + "frequency": "hourly", + "data[0]": "value", + "facets[respondent][]": ERCOT_BA_CODE, + "facets[type][]": "TI", # Total Interchange + "start": start_ts.strftime("%Y-%m-%dT00"), + "end": end_ts.strftime("%Y-%m-%dT23"), + "sort[0][column]": "period", + "sort[0][direction]": "asc", + } + + try: + response = self._make_request("electricity/rto/region-data/data", params) + data = response.get("response", {}).get("data", []) + + if not data: + return pd.DataFrame(columns=["timestamp", "interchange_mw"]) + + records = [] + for item in data: + records.append( + { + "timestamp": pd.Timestamp( + item.get("period"), tz=ERCOT_TIMEZONE + ), + "interchange_mw": float(item.get("value", 0)), + } + ) + + return pd.DataFrame(records) + + except Exception as e: + logger.error(f"Failed to fetch EIA interchange data: {e}") + return pd.DataFrame(columns=["timestamp", "interchange_mw"]) + + +def _map_fuel_type(code: str) -> str: + """Map EIA fuel type code to readable name.""" + mapping = { + "COL": "coal", + "NG": "natural_gas", + "NUC": "nuclear", + "OIL": "oil", + "WAT": "hydro", + "WND": "wind", + "SUN": "solar", + "OTH": "other", + "UNK": "unknown", + } + return mapping.get(code.upper(), code.lower()) diff --git a/tinygrid/ercot/polling.py b/tinygrid/ercot/polling.py new file mode 100644 index 0000000..2c28460 --- /dev/null +++ b/tinygrid/ercot/polling.py @@ -0,0 +1,343 @@ +"""Polling utilities for real-time ERCOT data access. + +ERCOT does not provide WebSocket or streaming endpoints. For real-time +data access, polling is required. This module provides utilities to +efficiently poll the ERCOT API with: + +- Rate limit awareness (30 requests/minute) +- Configurable poll intervals +- Exponential backoff on errors +- Callback-based or generator patterns +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable, Generator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, TypeVar + +import pandas as pd + +from ..errors import GridError + +if TYPE_CHECKING: + from . import ERCOT + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + +# Minimum poll interval in seconds (to respect rate limits) +MIN_POLL_INTERVAL = 2.0 # ~30 requests per minute + +# Default poll interval in seconds +DEFAULT_POLL_INTERVAL = 60.0 # 1 minute + +# Maximum consecutive errors before stopping +MAX_CONSECUTIVE_ERRORS = 5 + + +@dataclass +class PollResult: + """Result of a single poll iteration.""" + + data: pd.DataFrame | None + timestamp: pd.Timestamp + success: bool + error: Exception | None = None + iteration: int = 0 + + +class ERCOTPoller: + """Utility for polling ERCOT data at regular intervals. + + Provides a convenient way to continuously fetch data from the ERCOT API + with proper rate limiting and error handling. + + Args: + client: ERCOT client instance + interval: Poll interval in seconds. Minimum 2 seconds to respect rate limits. + max_errors: Maximum consecutive errors before stopping. Default 5. + backoff_factor: Multiplier for exponential backoff on errors. Default 2.0. + max_backoff: Maximum backoff delay in seconds. Default 300. + + Example: + ```python + from tinygrid import ERCOT + from tinygrid.ercot.polling import ERCOTPoller + + ercot = ERCOT(auth=auth) + poller = ERCOTPoller(client=ercot, interval=60) + + # Using callback pattern + def handle_data(result): + if result.success: + print(f"Got {len(result.data)} rows at {result.timestamp}") + else: + print(f"Error: {result.error}") + + poller.poll( + method=ercot.get_spp, + callback=handle_data, + max_iterations=10, + ) + + # Using generator pattern + for result in poller.poll_iter(method=ercot.get_spp, max_iterations=10): + if result.success: + process_data(result.data) + ``` + """ + + def __init__( + self, + client: ERCOT, + interval: float = DEFAULT_POLL_INTERVAL, + max_errors: int = MAX_CONSECUTIVE_ERRORS, + backoff_factor: float = 2.0, + max_backoff: float = 300.0, + ) -> None: + """Initialize the poller. + + Args: + client: ERCOT client instance + interval: Poll interval in seconds (minimum 2 seconds) + max_errors: Maximum consecutive errors before stopping + backoff_factor: Multiplier for exponential backoff + max_backoff: Maximum backoff delay in seconds + """ + self.client = client + self.interval = max(interval, MIN_POLL_INTERVAL) + self.max_errors = max_errors + self.backoff_factor = backoff_factor + self.max_backoff = max_backoff + + self._running = False + self._consecutive_errors = 0 + self._current_backoff = 0.0 + + def poll( + self, + method: Callable[..., pd.DataFrame], + callback: Callable[[PollResult], Any], + max_iterations: int | None = None, + **kwargs: Any, + ) -> None: + """Poll an ERCOT method continuously with a callback. + + Args: + method: ERCOT client method to call (e.g., client.get_spp) + callback: Function to call with each PollResult + max_iterations: Maximum number of iterations (None = infinite) + **kwargs: Arguments to pass to the method + + Example: + ```python + def handle_spp(result): + if result.success: + df = result.data + print(f"Latest SPP: {df['Price'].mean():.2f}") + + poller.poll( + method=ercot.get_spp, + callback=handle_spp, + market=Market.REAL_TIME_15_MIN, + ) + ``` + """ + self._running = True + iteration = 0 + + try: + while self._running: + if max_iterations is not None and iteration >= max_iterations: + break + + result = self._poll_once(method, iteration, **kwargs) + callback(result) + + if not result.success: + self._handle_error() + else: + self._reset_backoff() + + iteration += 1 + + # Check if we should stop due to too many errors + if self._consecutive_errors >= self.max_errors: + logger.error( + f"Stopping poller after {self.max_errors} consecutive errors" + ) + break + + # Wait for next poll + wait_time = self.interval + self._current_backoff + time.sleep(wait_time) + + finally: + self._running = False + + def poll_iter( + self, + method: Callable[..., pd.DataFrame], + max_iterations: int | None = None, + **kwargs: Any, + ) -> Generator[PollResult, None, None]: + """Poll an ERCOT method as a generator. + + Yields PollResult objects for each iteration. Use this pattern when + you want more control over the polling loop. + + Args: + method: ERCOT client method to call + max_iterations: Maximum number of iterations (None = infinite) + **kwargs: Arguments to pass to the method + + Yields: + PollResult for each poll iteration + + Example: + ```python + for result in poller.poll_iter(method=ercot.get_spp, max_iterations=5): + if result.success: + print(f"Iteration {result.iteration}: {len(result.data)} rows") + if some_condition: + break # Can exit early + ``` + """ + self._running = True + iteration = 0 + + try: + while self._running: + if max_iterations is not None and iteration >= max_iterations: + break + + result = self._poll_once(method, iteration, **kwargs) + yield result + + if not result.success: + self._handle_error() + else: + self._reset_backoff() + + iteration += 1 + + if self._consecutive_errors >= self.max_errors: + logger.error( + f"Stopping poller after {self.max_errors} consecutive errors" + ) + break + + wait_time = self.interval + self._current_backoff + time.sleep(wait_time) + + finally: + self._running = False + + def stop(self) -> None: + """Stop the poller gracefully.""" + self._running = False + + def _poll_once( + self, + method: Callable[..., pd.DataFrame], + iteration: int, + **kwargs: Any, + ) -> PollResult: + """Execute a single poll iteration.""" + timestamp = pd.Timestamp.now(tz="US/Central") + + try: + # For polling, we typically want the most recent data + # If start/end not provided, default to "today" + if "start" not in kwargs: + kwargs["start"] = "today" + + data = method(**kwargs) + + return PollResult( + data=data, + timestamp=timestamp, + success=True, + iteration=iteration, + ) + + except GridError as e: + logger.warning(f"Poll iteration {iteration} failed: {e}") + return PollResult( + data=None, + timestamp=timestamp, + success=False, + error=e, + iteration=iteration, + ) + except Exception as e: + logger.error(f"Unexpected error in poll iteration {iteration}: {e}") + return PollResult( + data=None, + timestamp=timestamp, + success=False, + error=e, + iteration=iteration, + ) + + def _handle_error(self) -> None: + """Handle a poll error by incrementing backoff.""" + self._consecutive_errors += 1 + self._current_backoff = min( + self._current_backoff * self.backoff_factor + self.interval, + self.max_backoff, + ) + logger.debug( + f"Poll error {self._consecutive_errors}, backoff: {self._current_backoff:.1f}s" + ) + + def _reset_backoff(self) -> None: + """Reset error tracking after successful poll.""" + self._consecutive_errors = 0 + self._current_backoff = 0.0 + + +def poll_latest( + client: ERCOT, + method: Callable[..., pd.DataFrame], + interval: float = DEFAULT_POLL_INTERVAL, + max_iterations: int | None = None, + **kwargs: Any, +) -> Generator[pd.DataFrame, None, None]: + """Simple generator for polling latest data. + + A convenience function for simple polling use cases. + + Args: + client: ERCOT client instance + method: Method to poll (e.g., client.get_spp) + interval: Poll interval in seconds + max_iterations: Maximum iterations (None = infinite) + **kwargs: Arguments to pass to the method + + Yields: + DataFrame for each successful poll (skips failures) + + Example: + ```python + from tinygrid import ERCOT + from tinygrid.ercot.polling import poll_latest + + ercot = ERCOT(auth=auth) + + for df in poll_latest(ercot, ercot.get_spp, interval=60, max_iterations=10): + print(f"Got {len(df)} rows") + # Process the data... + ``` + """ + poller = ERCOTPoller(client=client, interval=interval) + + for result in poller.poll_iter( + method=method, max_iterations=max_iterations, **kwargs + ): + if result.success and result.data is not None: + yield result.data diff --git a/tinygrid/utils/__init__.py b/tinygrid/utils/__init__.py index d36e424..7f12124 100644 --- a/tinygrid/utils/__init__.py +++ b/tinygrid/utils/__init__.py @@ -2,15 +2,29 @@ from .dates import date_chunks, format_api_date, parse_date, parse_date_range from .decorators import support_date_range, with_date_range +from .rate_limiter import ( + ERCOT_REQUESTS_PER_MINUTE, + AsyncRateLimiter, + RateLimiter, + rate_limited, +) from .tz import localize_with_dst, resolve_ambiguous_dst __all__ = [ + "ERCOT_REQUESTS_PER_MINUTE", + # Rate limiting + "AsyncRateLimiter", + "RateLimiter", + # Date utilities "date_chunks", "format_api_date", + # Timezone utilities "localize_with_dst", "parse_date", "parse_date_range", + "rate_limited", "resolve_ambiguous_dst", + # Decorators "support_date_range", "with_date_range", ] diff --git a/tinygrid/utils/rate_limiter.py b/tinygrid/utils/rate_limiter.py new file mode 100644 index 0000000..cb11789 --- /dev/null +++ b/tinygrid/utils/rate_limiter.py @@ -0,0 +1,285 @@ +"""Rate limiting utilities for API requests. + +ERCOT's Public API has a rate limit of 30 requests per minute. This module +provides a token bucket rate limiter to proactively enforce this limit and +avoid 429 errors. +""" + +from __future__ import annotations + +import asyncio +import logging +import threading +import time +from collections.abc import Callable +from typing import Any, TypeVar + +logger = logging.getLogger(__name__) + +# Default rate limit for ERCOT API (30 requests per minute) +ERCOT_REQUESTS_PER_MINUTE = 30 +ERCOT_MIN_INTERVAL = 60.0 / ERCOT_REQUESTS_PER_MINUTE # ~2 seconds + +T = TypeVar("T") + + +class RateLimiter: + """Thread-safe token bucket rate limiter for API requests. + + Implements a token bucket algorithm where: + - Tokens are added at a fixed rate (requests_per_minute / 60 per second) + - Each request consumes one token + - If no tokens are available, the request blocks until one is available + + This proactively prevents rate limit errors (HTTP 429) by throttling + requests before they hit the API. + + Args: + requests_per_minute: Maximum requests allowed per minute. Defaults to 30. + burst_size: Maximum tokens that can accumulate (burst capacity). + Defaults to requests_per_minute (allows full burst at start). + + Example: + ```python + from tinygrid.utils.rate_limiter import RateLimiter + + limiter = RateLimiter(requests_per_minute=30) + + # Use as context manager + with limiter: + response = make_api_request() + + # Or call acquire/release manually + limiter.acquire() + try: + response = make_api_request() + finally: + limiter.release() + ``` + """ + + def __init__( + self, + requests_per_minute: float = ERCOT_REQUESTS_PER_MINUTE, + burst_size: float | None = None, + ) -> None: + """Initialize the rate limiter. + + Args: + requests_per_minute: Maximum requests per minute + burst_size: Maximum burst capacity (tokens). Defaults to requests_per_minute. + """ + self.requests_per_minute = requests_per_minute + self.burst_size = burst_size if burst_size is not None else requests_per_minute + + # Token bucket state + self._tokens = self.burst_size + self._last_update = time.monotonic() + self._lock = threading.Lock() + + # Calculate refill rate (tokens per second) + self._refill_rate = requests_per_minute / 60.0 + + def _refill_tokens(self) -> None: + """Refill tokens based on elapsed time.""" + now = time.monotonic() + elapsed = now - self._last_update + self._tokens = min(self.burst_size, self._tokens + elapsed * self._refill_rate) + self._last_update = now + + def acquire(self, timeout: float | None = None) -> bool: + """Acquire a token, blocking if necessary. + + Args: + timeout: Maximum time to wait for a token (seconds). None means wait forever. + + Returns: + True if token was acquired, False if timeout occurred + """ + deadline = None if timeout is None else time.monotonic() + timeout + + while True: + with self._lock: + self._refill_tokens() + + if self._tokens >= 1.0: + self._tokens -= 1.0 + return True + + # Calculate wait time for next token + wait_time = (1.0 - self._tokens) / self._refill_rate + + # Check timeout + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + wait_time = min(wait_time, remaining) + + # Wait for token refill + logger.debug(f"Rate limiter: waiting {wait_time:.2f}s for token") + time.sleep(wait_time) + + def release(self) -> None: + """Release is a no-op for token bucket (tokens are consumed, not borrowed).""" + pass + + def __enter__(self) -> RateLimiter: + """Context manager entry - acquires a token.""" + self.acquire() + return self + + def __exit__(self, *args: Any) -> None: + """Context manager exit.""" + self.release() + + @property + def available_tokens(self) -> float: + """Get the current number of available tokens.""" + with self._lock: + self._refill_tokens() + return self._tokens + + @property + def min_interval(self) -> float: + """Minimum interval between requests in seconds.""" + return 60.0 / self.requests_per_minute + + def reset(self) -> None: + """Reset the rate limiter to full capacity.""" + with self._lock: + self._tokens = self.burst_size + self._last_update = time.monotonic() + + +class AsyncRateLimiter: + """Async-compatible token bucket rate limiter. + + Same algorithm as RateLimiter but uses asyncio for non-blocking waits. + + Args: + requests_per_minute: Maximum requests allowed per minute + burst_size: Maximum burst capacity + + Example: + ```python + from tinygrid.utils.rate_limiter import AsyncRateLimiter + + limiter = AsyncRateLimiter(requests_per_minute=30) + + async def fetch_data(): + async with limiter: + return await make_api_request() + ``` + """ + + def __init__( + self, + requests_per_minute: float = ERCOT_REQUESTS_PER_MINUTE, + burst_size: float | None = None, + ) -> None: + """Initialize the async rate limiter.""" + self.requests_per_minute = requests_per_minute + self.burst_size = burst_size if burst_size is not None else requests_per_minute + + self._tokens = self.burst_size + self._last_update = time.monotonic() + self._lock = asyncio.Lock() + self._refill_rate = requests_per_minute / 60.0 + + def _refill_tokens(self) -> None: + """Refill tokens based on elapsed time.""" + now = time.monotonic() + elapsed = now - self._last_update + self._tokens = min(self.burst_size, self._tokens + elapsed * self._refill_rate) + self._last_update = now + + async def acquire(self, timeout: float | None = None) -> bool: + """Acquire a token, awaiting if necessary. + + Args: + timeout: Maximum time to wait for a token (seconds) + + Returns: + True if token was acquired, False if timeout occurred + """ + deadline = None if timeout is None else time.monotonic() + timeout + + while True: + async with self._lock: + self._refill_tokens() + + if self._tokens >= 1.0: + self._tokens -= 1.0 + return True + + wait_time = (1.0 - self._tokens) / self._refill_rate + + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + wait_time = min(wait_time, remaining) + + logger.debug(f"Async rate limiter: waiting {wait_time:.2f}s for token") + await asyncio.sleep(wait_time) + + async def release(self) -> None: + """Release is a no-op for token bucket.""" + pass + + async def __aenter__(self) -> AsyncRateLimiter: + """Async context manager entry.""" + await self.acquire() + return self + + async def __aexit__(self, *args: Any) -> None: + """Async context manager exit.""" + await self.release() + + @property + def available_tokens(self) -> float: + """Get the current number of available tokens (sync access).""" + self._refill_tokens() + return self._tokens + + def reset(self) -> None: + """Reset the rate limiter to full capacity.""" + self._tokens = self.burst_size + self._last_update = time.monotonic() + + +def rate_limited( + limiter: RateLimiter | None = None, + requests_per_minute: float = ERCOT_REQUESTS_PER_MINUTE, +) -> Callable[[Callable[..., T]], Callable[..., T]]: + """Decorator to apply rate limiting to a function. + + Args: + limiter: Existing RateLimiter to use. If None, creates a new one. + requests_per_minute: Rate limit if creating new limiter + + Returns: + Decorated function that respects rate limits + + Example: + ```python + from tinygrid.utils.rate_limiter import rate_limited + + @rate_limited(requests_per_minute=30) + def make_api_call(endpoint: str): + return requests.get(endpoint) + ``` + """ + _limiter = limiter or RateLimiter(requests_per_minute=requests_per_minute) + + def decorator(func: Callable[..., T]) -> Callable[..., T]: + def wrapper(*args: Any, **kwargs: Any) -> T: + with _limiter: + return func(*args, **kwargs) + + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + return wrapper + + return decorator From 623b2d94d2874e93b677f24ccb857a19aaa37cad Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Dec 2025 02:31:01 +0000 Subject: [PATCH 3/3] Test: Add tests for non-historical API paths and poller edge cases Co-authored-by: kvkenyon --- tests/test_ercot_api_methods.py | 120 +++++++++++ tests/test_ercot_client.py | 140 +++++++++++++ tests/test_ercot_dashboard.py | 354 ++++++++++++++++++++++++++++++++ tests/test_polling.py | 103 ++++++++++ 4 files changed, 717 insertions(+) diff --git a/tests/test_ercot_api_methods.py b/tests/test_ercot_api_methods.py index 7535527..f4ae757 100644 --- a/tests/test_ercot_api_methods.py +++ b/tests/test_ercot_api_methods.py @@ -266,3 +266,123 @@ def test_solar_forecast_hourly_by_region(self, mixin): call_args = mixin._mock_archive.fetch_historical.call_args assert "np4-745-cd" in call_args[1]["endpoint"] + + +class NonHistoricalMixin(ERCOTAPIMixin): + """Testable mixin that always uses non-historical (live API) path.""" + + def __init__(self): + self._mock_archive = MagicMock() + self._mock_archive.fetch_historical = MagicMock(return_value=pd.DataFrame()) + self._endpoint_calls = [] + + def _get_archive(self): + return self._mock_archive + + def _needs_historical(self, date, data_type): + return False # Always use live API for testing + + # Mock the pyercot endpoint methods + def get_wpp_hourly_average_actual_forecast(self, **kwargs): + self._endpoint_calls.append(("wpp_hourly", kwargs)) + return pd.DataFrame() + + def get_wpp_hourly_actual_forecast_geo(self, **kwargs): + self._endpoint_calls.append(("wpp_hourly_geo", kwargs)) + return pd.DataFrame() + + def get_wpp_actual_5min_avg_values(self, **kwargs): + self._endpoint_calls.append(("wpp_5min", kwargs)) + return pd.DataFrame() + + def get_wpp_actual_5min_avg_values_geo(self, **kwargs): + self._endpoint_calls.append(("wpp_5min_geo", kwargs)) + return pd.DataFrame() + + def get_spp_hourly_average_actual_forecast(self, **kwargs): + self._endpoint_calls.append(("spp_hourly", kwargs)) + return pd.DataFrame() + + def get_spp_hourly_actual_forecast_geo(self, **kwargs): + self._endpoint_calls.append(("spp_hourly_geo", kwargs)) + return pd.DataFrame() + + def get_spp_actual_5min_avg_values(self, **kwargs): + self._endpoint_calls.append(("spp_5min", kwargs)) + return pd.DataFrame() + + def get_spp_actual_5min_avg_values_geo(self, **kwargs): + self._endpoint_calls.append(("spp_5min_geo", kwargs)) + return pd.DataFrame() + + +class TestNonHistoricalWindForecast: + """Tests for get_wind_forecast using live API (non-historical) path.""" + + @pytest.fixture + def mixin(self): + return NonHistoricalMixin() + + def test_wind_forecast_hourly_live(self, mixin): + """Test wind forecast hourly uses live endpoint.""" + mixin.get_wind_forecast(start="today", resolution="hourly") + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "wpp_hourly" + + def test_wind_forecast_hourly_by_region_live(self, mixin): + """Test wind forecast hourly by region uses live endpoint.""" + mixin.get_wind_forecast(start="today", resolution="hourly", by_region=True) + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "wpp_hourly_geo" + + def test_wind_forecast_5min_live(self, mixin): + """Test wind forecast 5min uses live endpoint.""" + mixin.get_wind_forecast(start="today", resolution="5min") + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "wpp_5min" + + def test_wind_forecast_5min_by_region_live(self, mixin): + """Test wind forecast 5min by region uses live endpoint.""" + mixin.get_wind_forecast(start="today", resolution="5min", by_region=True) + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "wpp_5min_geo" + + +class TestNonHistoricalSolarForecast: + """Tests for get_solar_forecast using live API (non-historical) path.""" + + @pytest.fixture + def mixin(self): + return NonHistoricalMixin() + + def test_solar_forecast_hourly_live(self, mixin): + """Test solar forecast hourly uses live endpoint.""" + mixin.get_solar_forecast(start="today", resolution="hourly") + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "spp_hourly" + + def test_solar_forecast_hourly_by_region_live(self, mixin): + """Test solar forecast hourly by region uses live endpoint.""" + mixin.get_solar_forecast(start="today", resolution="hourly", by_region=True) + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "spp_hourly_geo" + + def test_solar_forecast_5min_live(self, mixin): + """Test solar forecast 5min uses live endpoint.""" + mixin.get_solar_forecast(start="today", resolution="5min") + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "spp_5min" + + def test_solar_forecast_5min_by_region_live(self, mixin): + """Test solar forecast 5min by region uses live endpoint.""" + mixin.get_solar_forecast(start="today", resolution="5min", by_region=True) + + assert len(mixin._endpoint_calls) == 1 + assert mixin._endpoint_calls[0][0] == "spp_5min_geo" diff --git a/tests/test_ercot_client.py b/tests/test_ercot_client.py index 83759dd..1ff42cb 100644 --- a/tests/test_ercot_client.py +++ b/tests/test_ercot_client.py @@ -256,3 +256,143 @@ def test_to_dataframe_empty(self): df = client._to_dataframe([[1, 2]], []) assert not df.empty assert df.shape == (1, 2) + + def test_get_client_with_no_auth(self): + """Test _get_client without authentication.""" + client = ERCOTBase() + + # Get client + c1 = client._get_client() + assert c1 is not None + + # Second call reuses client + c2 = client._get_client() + assert c1 is c2 + + def test_get_rate_limiter_disabled(self): + """Test _get_rate_limiter when rate limiting is disabled.""" + client = ERCOTBase(rate_limit_enabled=False) + + limiter = client._get_rate_limiter() + assert limiter is None + + def test_get_rate_limiter_enabled(self): + """Test _get_rate_limiter when rate limiting is enabled.""" + client = ERCOTBase(rate_limit_enabled=True) + + limiter = client._get_rate_limiter() + assert limiter is not None + + # Second call returns same instance + limiter2 = client._get_rate_limiter() + assert limiter is limiter2 + + def test_products_to_dataframe_raw_list(self): + """Test _products_to_dataframe with raw list input.""" + client = ERCOTBase() + + # Raw list response + df = client._products_to_dataframe([{"id": 1}, {"id": 2}]) + assert not df.empty + assert len(df) == 2 + + def test_products_to_dataframe_hal_format(self): + """Test _products_to_dataframe with HAL format.""" + client = ERCOTBase() + + # HAL format: {"_embedded": {"products": [...]}} + df = client._products_to_dataframe( + {"_embedded": {"products": [{"id": 1}, {"id": 2}]}} + ) + assert not df.empty + assert len(df) == 2 + + def test_products_to_dataframe_additional_properties(self): + """Test _products_to_dataframe with additional_properties.""" + client = ERCOTBase() + + # additional_properties with products key + df = client._products_to_dataframe( + {"additional_properties": {"products": [{"id": 1}]}} + ) + assert not df.empty + assert len(df) == 1 + + # additional_properties with _embedded key + df = client._products_to_dataframe( + {"additional_properties": {"_embedded": {"products": [{"id": 2}]}}} + ) + assert not df.empty + assert len(df) == 1 + + def test_products_to_dataframe_none(self): + """Test _products_to_dataframe with None input.""" + client = ERCOTBase() + df = client._products_to_dataframe(None) + assert df.empty + + def test_products_to_dataframe_non_dict(self): + """Test _products_to_dataframe with non-dict, non-list input.""" + client = ERCOTBase() + df = client._products_to_dataframe("not a dict or list") + assert df.empty + + def test_extract_response_data_to_dict_failure(self): + """Test _extract_response_data when to_dict() fails.""" + client = ERCOTBase() + + class FailingToDict: + def to_dict(self): + raise ValueError("Failed") + + # Should not raise, fallback to other methods + result = client._extract_response_data(FailingToDict()) + assert result == {} + + def test_extract_response_data_data_to_dict_failure(self): + """Test _extract_response_data when data.to_dict() fails.""" + client = ERCOTBase() + + class FailingDataToDict: + def to_dict(self): + raise ValueError("Failed") + + class ReportWithFailingData: + data = FailingDataToDict() + + result = client._extract_response_data(ReportWithFailingData()) + assert result == {} + + def test_call_endpoint_raw_none_response(self): + """Test _call_endpoint_raw with None response.""" + client = ERCOTBase() + mock_module = MagicMock() + mock_module.sync.return_value = None + + result = client._call_endpoint_raw(mock_module, "test") + assert result == {} + + def test_context_manager(self): + """Test ERCOTBase context manager enter/exit.""" + client = ERCOTBase() + + with client as c: + assert c is client + assert c._entered_client is not None + + # After exiting, _entered_client should be None + assert client._entered_client is None + + def test_should_use_historical(self): + """Test _should_use_historical method.""" + import pandas as pd + + client = ERCOTBase() + + # Date far in the past should use historical + old_date = pd.Timestamp("2020-01-01", tz="US/Central") + assert client._should_use_historical(old_date) is True + + # Recent date should not use historical + recent_date = pd.Timestamp.now(tz="US/Central") - pd.Timedelta(days=10) + assert client._should_use_historical(recent_date) is False diff --git a/tests/test_ercot_dashboard.py b/tests/test_ercot_dashboard.py index b8cf1c0..2cc0274 100644 --- a/tests/test_ercot_dashboard.py +++ b/tests/test_ercot_dashboard.py @@ -606,3 +606,357 @@ def test_get_capacity_committed_success(self, mock_fetch, mixin_instance): assert isinstance(df, pd.DataFrame) assert len(df) == 2 assert "hour" in df.columns + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_exception_handling(self, mock_fetch, mixin_instance): + """Test get_status handles exceptions gracefully.""" + mock_fetch.return_value = {"current": {"invalid": "data"}} + # Should not raise - returns valid status with defaults + status = mixin_instance.get_status() + assert isinstance(status, GridStatus) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_non_normal_condition_with_no_message( + self, mock_fetch, mixin_instance + ): + """Test get_status builds message for non-normal conditions.""" + mock_fetch.return_value = { + "current": { + "condition": "watch", + "demand": 50000, + "capacity": 70000, + } + } + status = mixin_instance.get_status() + assert status.condition == GridCondition.WATCH + assert "watch" in status.message.lower() + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_with_prc_and_renewable(self, mock_fetch, mixin_instance): + """Test get_status extracts PRC and renewable data.""" + mock_fetch.return_value = { + "current": { + "condition": "normal", + "demand": 50000, + "capacity": 70000, + "prc": 5000, + "windOutput": 12000, + "solarOutput": 8000, + "peakForecast": 75000, + } + } + status = mixin_instance.get_status() + assert status.prc == 5000.0 + assert status.wind_output == 12000.0 + assert status.solar_output == 8000.0 + assert status.peak_forecast == 75000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_failure_returns_empty(self, mock_fetch, mixin_instance): + """Test get_fuel_mix returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_failure_as_list(self, mock_fetch, mixin_instance): + """Test get_fuel_mix returns empty list on failure with as_dataframe=False.""" + mock_fetch.return_value = None + result = mixin_instance.get_fuel_mix(as_dataframe=False) + assert isinstance(result, list) + assert len(result) == 0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_exception_handling(self, mock_fetch, mixin_instance): + """Test get_fuel_mix handles parsing exceptions.""" + # Return invalid data that will cause parsing exception + mock_fetch.return_value = {"data": "invalid"} + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_exception_as_list(self, mock_fetch, mixin_instance): + """Test get_fuel_mix returns empty list on exception with as_dataframe=False.""" + mock_fetch.return_value = {"data": "invalid"} + result = mixin_instance.get_fuel_mix(as_dataframe=False) + assert isinstance(result, list) + assert len(result) == 0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_empty_data(self, mock_fetch, mixin_instance): + """Test get_fuel_mix with empty data list.""" + mock_fetch.return_value = {"data": [], "lastUpdated": 1704067200000} + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_renewable_generation_failure(self, mock_fetch, mixin_instance): + """Test get_renewable_generation returns defaults on failure.""" + mock_fetch.return_value = None + status = mixin_instance.get_renewable_generation() + assert isinstance(status, RenewableStatus) + assert status.wind_mw == 0.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_renewable_generation_exception(self, mock_fetch, mixin_instance): + """Test get_renewable_generation handles exceptions gracefully.""" + # Return data that will cause parsing exception + mock_fetch.return_value = {"current": None} + status = mixin_instance.get_renewable_generation() + assert isinstance(status, RenewableStatus) + assert status.wind_mw == 0.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_supply_demand_failure(self, mock_fetch, mixin_instance): + """Test get_supply_demand returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_supply_demand() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_supply_demand_empty_data(self, mock_fetch, mixin_instance): + """Test get_supply_demand with empty data list.""" + mock_fetch.return_value = {"data": [], "lastUpdated": 1704067200000} + df = mixin_instance.get_supply_demand() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_supply_demand_exception(self, mock_fetch, mixin_instance): + """Test get_supply_demand handles exceptions gracefully.""" + mock_fetch.return_value = {"data": None} + df = mixin_instance.get_supply_demand() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_daily_prices_failure(self, mock_fetch, mixin_instance): + """Test get_daily_prices returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_daily_prices() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_daily_prices_empty_data(self, mock_fetch, mixin_instance): + """Test get_daily_prices with empty data list.""" + mock_fetch.return_value = {"data": [], "lastUpdated": 1704067200000} + df = mixin_instance.get_daily_prices() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_daily_prices_exception(self, mock_fetch, mixin_instance): + """Test get_daily_prices handles exceptions gracefully.""" + mock_fetch.return_value = {"data": None} + df = mixin_instance.get_daily_prices() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_system_wide_demand_failure(self, mock_fetch, mixin_instance): + """Test get_system_wide_demand returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_system_wide_demand() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_system_wide_demand_empty_data(self, mock_fetch, mixin_instance): + """Test get_system_wide_demand with empty data.""" + mock_fetch.return_value = {"current": {}, "hourly": []} + df = mixin_instance.get_system_wide_demand() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_system_wide_demand_exception(self, mock_fetch, mixin_instance): + """Test get_system_wide_demand handles exceptions gracefully.""" + mock_fetch.return_value = {"invalid": "data"} + df = mixin_instance.get_system_wide_demand() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_energy_storage_resources_failure(self, mock_fetch, mixin_instance): + """Test get_energy_storage_resources returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_energy_storage_resources() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_energy_storage_resources_no_esr(self, mock_fetch, mixin_instance): + """Test get_energy_storage_resources when ESR data not present.""" + mock_fetch.return_value = {"current": {"demand": 50000}} + df = mixin_instance.get_energy_storage_resources() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_energy_storage_resources_exception(self, mock_fetch, mixin_instance): + """Test get_energy_storage_resources handles exceptions gracefully.""" + mock_fetch.return_value = {"current": {"esr": "invalid"}} + df = mixin_instance.get_energy_storage_resources() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_capacity_committed_failure(self, mock_fetch, mixin_instance): + """Test get_capacity_committed returns empty DataFrame on failure.""" + mock_fetch.return_value = None + df = mixin_instance.get_capacity_committed() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_capacity_committed_empty_data(self, mock_fetch, mixin_instance): + """Test get_capacity_committed with empty data list.""" + mock_fetch.return_value = {"data": [], "lastUpdated": 1704067200000} + df = mixin_instance.get_capacity_committed() + assert isinstance(df, pd.DataFrame) + assert df.empty + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_capacity_committed_exception(self, mock_fetch, mixin_instance): + """Test get_capacity_committed handles exceptions gracefully.""" + mock_fetch.return_value = {"data": None} + df = mixin_instance.get_capacity_committed() + assert isinstance(df, pd.DataFrame) + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_status_with_alternative_field_names(self, mock_fetch, mixin_instance): + """Test get_status parses alternative field names correctly.""" + mock_fetch.return_value = { + "status": "normal", + "load": 55000, + "totalCapacity": 80000, + "operatingReserves": 25000, + "wind": 15000, + "solar": 9000, + "peak": 78000, + "physicalResponsive": 4500, + "timestamp": 1704067200000, + "alert": "Test alert message", + } + status = mixin_instance.get_status() + assert status.current_load == 55000.0 + assert status.capacity == 80000.0 + assert status.reserves == 25000.0 + + +class TestFetchJsonGenericException: + """Tests for _fetch_json generic exception handling.""" + + @patch("tinygrid.ercot.dashboard.httpx.Client") + def test_fetch_json_generic_exception(self, mock_client_class): + """Test generic exception returns None.""" + from unittest.mock import MagicMock + + mock_client = MagicMock() + mock_client.get.side_effect = Exception("Unexpected error") + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=None) + mock_client_class.return_value = mock_client + + result = _fetch_json("https://example.com/api") + + assert result is None + + +class TestDashboardAlternativeParsing: + """Tests for alternative data structure parsing in dashboard methods.""" + + @pytest.fixture + def mixin_instance(self): + """Create a test instance with the mixin.""" + + class TestClass(ERCOTDashboardMixin): + pass + + return TestClass() + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_with_fuelMix_key(self, mock_fetch, mixin_instance): + """Test get_fuel_mix parsing with 'fuelMix' key.""" + mock_fetch.return_value = { + "fuelMix": [ + {"fuelType": "natural_gas", "generation": 30000}, + ], + "timestamp": 1704067200000, + } + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + assert len(df) == 1 + assert df.iloc[0]["fuel_type"] == "natural_gas" + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_fuel_mix_with_mw_key(self, mock_fetch, mixin_instance): + """Test get_fuel_mix parsing with 'mw' key.""" + mock_fetch.return_value = { + "data": [ + {"type": "coal", "mw": 20000, "percentage": 35}, + ], + "lastUpdated": 1704067200000, + } + df = mixin_instance.get_fuel_mix() + assert isinstance(df, pd.DataFrame) + assert len(df) == 1 + assert df.iloc[0]["generation_mw"] == 20000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_supply_demand_with_hourly_key(self, mock_fetch, mixin_instance): + """Test get_supply_demand parsing with 'hourly' key.""" + mock_fetch.return_value = { + "hourly": [ + {"hourEnding": 1, "load": 45000, "capacity": 60000, "reserves": 15000}, + ], + "lastUpdated": 1704067200000, + } + df = mixin_instance.get_supply_demand() + assert isinstance(df, pd.DataFrame) + assert len(df) == 1 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_daily_prices_with_prices_key(self, mock_fetch, mixin_instance): + """Test get_daily_prices parsing with 'prices' key.""" + mock_fetch.return_value = { + "prices": [ + {"sp": "HB_HOUSTON", "spp": 30.50, "peakPrice": 45.0, "avgPrice": 28.0}, + ], + "lastUpdated": 1704067200000, + } + df = mixin_instance.get_daily_prices() + assert isinstance(df, pd.DataFrame) + assert len(df) == 1 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_renewable_with_alternative_keys(self, mock_fetch, mixin_instance): + """Test get_renewable_generation parsing with alternative keys.""" + mock_fetch.return_value = { + "current": { + "wind": 18000, + "solar": 8000, + "windFcst": 19000, + "solarFcst": 7500, + "windCap": 35000, + "solarCap": 20000, + }, + "lastUpdated": 1704067200000, + } + status = mixin_instance.get_renewable_generation() + assert status.wind_mw == 18000.0 + assert status.solar_mw == 8000.0 + assert status.wind_forecast_mw == 19000.0 + + @patch("tinygrid.ercot.dashboard._fetch_json") + def test_get_capacity_committed_with_hourly_key(self, mock_fetch, mixin_instance): + """Test get_capacity_committed parsing with 'hourly' key.""" + mock_fetch.return_value = { + "hourly": [ + {"hourEnding": 1, "supply": 60000, "capacity": 70000}, + ], + "lastUpdated": 1704067200000, + } + df = mixin_instance.get_capacity_committed() + assert isinstance(df, pd.DataFrame) + assert len(df) == 1 diff --git a/tests/test_polling.py b/tests/test_polling.py index 7940e61..8c2425e 100644 --- a/tests/test_polling.py +++ b/tests/test_polling.py @@ -412,3 +412,106 @@ def test_poll_latest_passes_kwargs(self, mock_sleep): call_kwargs = mock_method.call_args[1] assert call_kwargs["custom_param"] == "value" + + +class TestPollerEdgeCases: + """Tests for edge cases in poller behavior.""" + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_callback_stops_on_max_errors(self, mock_sleep): + """Test poll with callback stops after max consecutive errors.""" + mock_client = MagicMock() + mock_method = MagicMock(side_effect=GridAPIError("Error", status_code=500)) + callback_results = [] + + def callback(result): + callback_results.append(result) + + poller = ERCOTPoller(client=mock_client, max_errors=3) + poller.poll(method=mock_method, callback=callback, max_iterations=10) + + # Should stop after 3 errors + assert len(callback_results) == 3 + assert all(not r.success for r in callback_results) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_with_explicit_start_arg(self, mock_sleep): + """Test poll_once does not override explicit start argument.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client) + poller._poll_once(mock_method, iteration=0, start="2024-01-01") + + call_kwargs = mock_method.call_args[1] + assert call_kwargs["start"] == "2024-01-01" + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_can_be_stopped_early(self, mock_sleep): + """Test poll_iter can be stopped early via stop().""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client) + + results = [] + for i, result in enumerate(poller.poll_iter(method=mock_method)): + results.append(result) + if i >= 2: + poller.stop() + + assert len(results) == 3 + assert not poller._running + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_uses_correct_wait_time(self, mock_sleep): + """Test poll_iter uses correct wait time between iterations.""" + mock_client = MagicMock() + mock_method = MagicMock(return_value=pd.DataFrame()) + + poller = ERCOTPoller(client=mock_client, interval=30.0) + + list(poller.poll_iter(method=mock_method, max_iterations=2)) + + # Sleep is called after each iteration except possibly the last + # For 2 iterations, we expect sleeps between them + assert mock_sleep.call_count >= 1 + # Verify the interval is correct + mock_sleep.assert_called_with(30.0) + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_iter_adds_backoff_to_wait_time(self, mock_sleep): + """Test poll_iter adds backoff to wait time on errors.""" + mock_client = MagicMock() + mock_method = MagicMock( + side_effect=[ + GridAPIError("Error", status_code=500), + pd.DataFrame(), + ] + ) + + poller = ERCOTPoller(client=mock_client, interval=10.0, max_errors=5) + + list(poller.poll_iter(method=mock_method, max_iterations=2)) + + # First sleep should include backoff + first_sleep_call = mock_sleep.call_args_list[0][0][0] + assert first_sleep_call > 10.0 # interval + backoff + + @patch("tinygrid.ercot.polling.time.sleep") + def test_poll_latest_skips_none_data(self, mock_sleep): + """Test poll_latest skips results with None data.""" + mock_client = MagicMock() + # Returns DataFrame, but empty result still has data set + mock_method = MagicMock(return_value=pd.DataFrame({"col": [1]})) + + results = list( + poll_latest( + client=mock_client, + method=mock_method, + max_iterations=2, + ) + ) + + assert len(results) == 2 + assert all(len(r) == 1 for r in results)