From 4458d9af070d84345cb2537d284ba3de0b6cf5fe Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 9 Jun 2026 15:54:10 +0300 Subject: [PATCH 1/2] refactor: fred --- Makefile | 4 +- src/_schemas.py | 8 + src/helpers/fred.py | 1768 +---------------- .../func_fred_fetch}/Makefile | 22 +- src/orchestration/func_fred_fetch/main.py | 33 + .../func_fred_fetch}/requirements.txt | 6 +- .../func_fred_update}/Makefile | 22 +- src/orchestration/func_fred_update/main.py | 37 + .../func_fred_update}/requirements.txt | 5 +- src/questions/fred/fetch/main.py | 433 ---- src/questions/fred/update_questions/main.py | 114 -- src/sources/_fred_questions.py | 1766 ++++++++++++++++ src/sources/_metadata.py | 5 + src/sources/fred.py | 412 +++- src/tests/conftest.py | 108 + src/tests/test_fred.py | 503 ++++- 16 files changed, 2884 insertions(+), 2362 deletions(-) rename src/{questions/fred/fetch => orchestration/func_fred_fetch}/Makefile (63%) create mode 100644 src/orchestration/func_fred_fetch/main.py rename src/{questions/fred/fetch => orchestration/func_fred_fetch}/requirements.txt (83%) rename src/{questions/fred/update_questions => orchestration/func_fred_update}/Makefile (64%) create mode 100644 src/orchestration/func_fred_update/main.py rename src/{questions/fred/update_questions => orchestration/func_fred_update}/requirements.txt (55%) delete mode 100644 src/questions/fred/fetch/main.py delete mode 100644 src/questions/fred/update_questions/main.py create mode 100644 src/sources/_fred_questions.py diff --git a/Makefile b/Makefile index 35c9b03e..7358251b 100644 --- a/Makefile +++ b/Makefile @@ -183,10 +183,10 @@ wikipedia-update-questions: fred: fred-fetch fred-update-questions fred-fetch: - $(MAKE) -C src/questions/fred/fetch || echo "* $@" >> $(MAKE_FAILURE_LOG) + $(MAKE) -C src/orchestration/func_fred_fetch || echo "* $@" >> $(MAKE_FAILURE_LOG) fred-update-questions: - $(MAKE) -C src/questions/fred/update_questions || echo "* $@" >> $(MAKE_FAILURE_LOG) + $(MAKE) -C src/orchestration/func_fred_update || echo "* $@" >> $(MAKE_FAILURE_LOG) dbnomics: dbnomics-fetch dbnomics-update-questions diff --git a/src/_schemas.py b/src/_schemas.py index e8ac7064..14320ae8 100644 --- a/src/_schemas.py +++ b/src/_schemas.py @@ -131,6 +131,14 @@ class Config: coerce = True +class FredFetchFrame(QuestionFrame): + """Output of FredSource.fetch(). QuestionFrame plus transient fields for update().""" + + fetch_datetime: Series[str] + probability: Series[object] = pa.Field(nullable=True) + resolutions: Series[object] # list[dict] per row: [{id, date, value}, ...] + + class AcledResolutionFrame(pa.DataFrameModel): """ACLED-specific: aggregated events by country and date. diff --git a/src/helpers/fred.py b/src/helpers/fred.py index 877d9666..33c90b21 100644 --- a/src/helpers/fred.py +++ b/src/helpers/fred.py @@ -1,4 +1,10 @@ -"""FRED Question List.""" +"""FRED constants — thin re-export shim over the lightweight sources._metadata layer. + +The canonical FRED question list and nullification data live in +``sources._metadata`` (the predefined series under the ``questions`` key). This +module re-exports them for backwards-compat with existing import sites, without +pulling the heavy ``sources.fred`` module. +""" import os import sys @@ -11,1762 +17,4 @@ SOURCE_INTRO = _META["source_intro"] RESOLUTION_CRITERIA = _META["resolution_criteria"] NULLIFIED_IDS = [nq.id for nq in _META["nullified_questions"]] - -# flake8: noqa: B950 - -fred_questions = [ - { - "id": "AAA10Y", - "series_name": "Moody's Aaa Corporate Bond Yield compared to the 10-year Treasury yield", - }, - { - "id": "ANFCI", - "series_name": "the Chicago Fed's Adjusted National Financial Conditions Index", - }, - { - "id": "BAA10Y", - "series_name": "Moody's Seasoned Baa Corporate Bond Yield compared to the 10-year Treasury yield", - }, - { - "id": "BAMLC0A0CM", - "series_name": "the option-adjusted spread of the ICE BofA Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A0CMEY", - "series_name": "the effective yield of the ICE BofA Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A1CAAA", - "series_name": "the option-adjusted spread of securities with an investment grade rating of AAA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A1CAAAEY", - "series_name": "the effective yield of securities with an investment grade rating of AAA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A2CAA", - "series_name": "the option-adjusted spread of securities with an investment grade rating of AA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A2CAAEY", - "series_name": "the effective yield of securities with an investment grade rating of AA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A3CA", - "series_name": "the option-adjusted spread of securities with an investment grade rating of A in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A3CAEY", - "series_name": "the effective yield of securities with an investment grade rating of A in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A4CBBB", - "series_name": "the option-adjusted spread of securities with an investment grade rating of BBB in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC0A4CBBBEY", - "series_name": "the effective yield of securities with an investment grade rating of BBB in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLC4A0C710YEY", - "series_name": "the effective yield of securities with a remaining term to maturity of 7-10 years in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLCC0A0CMTRIV", - "series_name": "the total return of the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", - }, - { - "id": "BAMLEMCBPIOAS", - "series_name": "the option-adjusted spread for the ICE BofA Emerging Markets Corporate Plus Index, which tracks the performance of emerging markets non-sovereign debt within major domestic and Eurobond markets,", - }, - { - "id": "BAMLEMHBHYCRPIOAS", - "series_name": "the option-adjusted spread for the ICE BofA High Yield Emerging Markets Corporate Plus Index, which tracks the performance of emerging markets securities rated BB1 or lower within major domestic and Eurobond markets,", - }, - { - "id": "BAMLH0A0HYM2", - "series_name": "the option-adjusted spread for the ICE BofA US High Yield Index, which tracks the performance of corporate debt denominated below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A0HYM2EY", - "series_name": "the effective yield of the ICE BofA US High Yield Index, which tracks the performance of corporate debt denominated below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A1HYBB", - "series_name": "the option-adjusted spread of securities with an investment grade rating of BB in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A1HYBBEY", - "series_name": "the effective yield of securities with an investment grade rating of BB in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A2HYB", - "series_name": "the option-adjusted spread of securities with an investment grade rating of B in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A2HYBEY", - "series_name": "the effective yield of securities with an investment grade rating of B in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A3HYC", - "series_name": "the option-adjusted spread of securities with an investment grade rating of CCC or below in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLH0A3HYCEY", - "series_name": "the effective yield of securities with an investment grade rating of CCC or below in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", - }, - { - "id": "BAMLHE00EHYIEY", - "series_name": "the effective yield of the ICE BofA Euro High Yield Index, which tracks the performance of below investment grade corporate debt issued in the euro domestic or eurobond markets,", - }, - { - "id": "BAMLHE00EHYIOAS", - "series_name": "the option-adjusted spread of the ICE BofA Euro High Yield Index, which tracks the performance of below investment grade corporate debt issued in the euro domestic or eurobond markets,", - }, - { - "id": "BAMLHYH0A0HYM2TRIV", - "series_name": "the total return of the ICE BofA US High Yield Index, which tracks the performance of below investment grade corporate debt publicly issued in the US domestic market,", - }, - { - "id": "CARACBW027SBOG", - "series_name": "the total dollar amount representing all automobile loans made by commercial banks in the US", - }, - {"id": "CASACBW027SBOG", "series_name": "the cash assets of all commercial US banks"}, - {"id": "CBBTCUSD", "series_name": "the price of Bitcoin, as measured by Coinbase,"}, - {"id": "CC4WSA", "series_name": "the 4-week moving average of insured unemployment claims"}, - { - "id": "CCLACBW027SBOG", - "series_name": "the amount of money representing all credit card loans and other revolving plans made by commercial banks in the US", - }, - {"id": "CCSA", "series_name": "the number of insured unemployment claims"}, - { - "id": "CREACBW027SBOG", - "series_name": "the amount of money representing all commercial real estate loans made by commercial banks in the US", - }, - { - "id": "D2WLTGAL", - "series_name": "the amount of money held by the US Treasury in its general account at the Federal Reserve Bank of New York", - }, - {"id": "DAAA", "series_name": "Moody's Seasoned Aaa Corporate Bond Yield"}, - {"id": "DBAA", "series_name": "Moody's Seasoned Baa Corporate Bond Yield"}, - {"id": "DCOILBRENTEU", "series_name": "the price of Brent crude oil"}, - { - "id": "DCOILWTICO", - "series_name": "the price of West Texas Intermediate (WTI - Cushing) crude oil", - }, - {"id": "DEXCAUS", "series_name": "the spot exchange rate of Canadian dollars to US dollars"}, - { - "id": "DEXCHUS", - "series_name": "the spot exchange rate of Chinese yuan renminbi to US dollars", - }, - {"id": "DEXJPUS", "series_name": "the spot exchange rate of Japanese yen to US dollars"}, - {"id": "DEXKOUS", "series_name": "the spot exchange rate of South Korean won to US dollars"}, - {"id": "DEXMXUS", "series_name": "the spot exchange rate of Mexican pesos to US dollars"}, - {"id": "DEXUSEU", "series_name": "the spot exchange rate of US dollars to euros"}, - {"id": "DEXUSUK", "series_name": "the spot exchange rate of US dollars to UK pound sterling"}, - { - "id": "DFEDTARL", - "series_name": "the lower limit of the target range of the federal funds rate (interest rate) set by the Federal Open Market Committee", - }, - { - "id": "DFEDTARU", - "series_name": "the upper limit of the target range of the federal funds rate (interest rate) set by the Federal Open Market Committee", - }, - { - "id": "DFF", - "series_name": "the effective federal funds rate (interest rate)", - }, - { - "id": "DFII10", - "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis and inflation-indexed,", - }, - { - "id": "DFII20", - "series_name": "the market yield on US treasury securities at 20-year constant maturity, quoted on an investment basis and inflation-indexed,", - }, - { - "id": "DFII30", - "series_name": "the market yield on US treasury securities at 30-year constant maturity, quoted on an investment basis and inflation-indexed,", - }, - { - "id": "DFII5", - "series_name": "the market yield on US treasury securities at 5-year constant maturity, quoted on an investment basis and inflation-indexed,", - }, - { - "id": "DGS1", - "series_name": "the market yield on US treasury securities at 1-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS10", - "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS1MO", - "series_name": "the market yield on US treasury securities at 1-month constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS2", - "series_name": "the market yield on US treasury securities at 2-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS20", - "series_name": "the market yield on US treasury securities at 20-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS3", - "series_name": "the market yield on US treasury securities at 3-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS30", - "series_name": "the market yield on US treasury securities at 30-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS3MO", - "series_name": "the market yield on US treasury securities at 3-month constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS5", - "series_name": "the market yield on US treasury securities at 5-year constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS6MO", - "series_name": "the market yield on US treasury securities at 6-month constant maturity, quoted on an investment basis,", - }, - { - "id": "DGS7", - "series_name": "the market yield on US treasury securities at 7-year constant maturity, quoted on an investment basis,", - }, - {"id": "DHHNGSP", "series_name": "the spot price of Henry Hub natural gas"}, - {"id": "DJIA", "series_name": "the Dow Jones Industrial Average"}, - { - "id": "DPCREDIT", - "series_name": "the discount rate for the Federal Reserve's primary credit discount window program", - }, - { - "id": "DPRIME", - "series_name": "the Federal Reserve's Bank Prime Loan Rate, the rate posted by a majority of top US commercial banks", - }, - { - "id": "DPSACBW027SBOG", - "series_name": "the amount of money representing deposits in all US commercial banks", - }, - { - "id": "DTB1YR", - "series_name": "the Federal Reserve's 1-year secondary market treasury bill rate", - }, - { - "id": "DTB3", - "series_name": "the Federal Reserve's 3-month secondary market treasury bill rate", - }, - { - "id": "DTB4WK", - "series_name": "the Federal Reserve's 4-week secondary market treasury bill rate", - }, - { - "id": "DTB6", - "series_name": "the Federal Reserve's 6-month secondary market treasury bill rate", - }, - { - "id": "DTWEXAFEGS", - "series_name": "the Nominal Advanced Foreign Economies US Dollar Index, a weighted average of the foreign exchange value of the US dollar against a subset of broad index currencies that are advanced foreign economies,", - }, - { - "id": "DTWEXBGS", - "series_name": "the Nominal Broad US Dollar Index, a weighted average of the foreign exchange value of the US dollar against currencies of a broad group of major US trading partners,", - }, - { - "id": "ECBASSETSW", - "series_name": "the amount of money representing all central bank assets for the euro area", - }, - { - "id": "ECBDFR", - "series_name": "the European Central Bank's deposit facility rate for the euro area", - }, - { - "id": "ECBESTRVOLWGTTRMDMNRT", - "series_name": "the euro short-term rate (volume-weighted trimmed mean), a measure of the borrowing costs of banks in the euro area,", - }, - { - "id": "EFFR", - "series_name": "the effective federal funds rate (interest rate) set by the Federal Reserve", - }, - { - "id": "EXPINF10YR", - "series_name": "the Federal Reserve Bank of Cleveland's 10-year expected inflation rate", - }, - { - "id": "EXPINF1YR", - "series_name": "the Federal Reserve Bank of Cleveland's 1-year expected inflation rate", - }, - { - "id": "EXPINF2YR", - "series_name": "the Federal Reserve Bank of Cleveland's 2-year expected inflation rate", - }, - { - "id": "EXPINF30YR", - "series_name": "the Federal Reserve Bank of Cleveland's 30-year expected inflation rate", - }, - { - "id": "EXPINF5YR", - "series_name": "the Federal Reserve Bank of Cleveland's 5-year expected inflation rate", - }, - {"id": "GASDESW", "series_name": "the average price of diesel in the US"}, - {"id": "GASREGW", "series_name": "the average price of regular gas in the US"}, - { - "id": "GVZCLS", - "series_name": "the Chicago Board Options Exchange's Gold ETF Volatility Index", - }, - { - "id": "H41RESPPALDKNWW", - "series_name": "the amount of money loaned as part of the Bank Term Funding Program", - }, - { - "id": "H41RESPPALDKXAWNWW", - "series_name": "the weekly average of the amount of money loaned as part of the Bank Term Funding Program", - }, - {"id": "IC4WSA", "series_name": "the 4-week moving average of initial unemployment claims"}, - {"id": "ICSA", "series_name": "the weekly number of initial unemployment claims"}, - {"id": "IHLIDXUS", "series_name": "the number of US job postings on Indeed"}, - { - "id": "IHLIDXUSTPSOFTDEVE", - "series_name": "the number of US software development job postings on Indeed", - }, - {"id": "IORB", "series_name": "the Federal Reserve's interest rate on reserve balances"}, - { - "id": "IUDSOIA", - "series_name": "the daily Sterling Overnight Index Average, the interest rate applied to bank transactions in the British Sterling Market during off hours,", - }, - { - "id": "MMTY", - "series_name": "the yield on money market investments based on US treasury obligations", - }, - {"id": "MORTGAGE15US", "series_name": "the 15-year fixed rate mortgage average in the US"}, - {"id": "MORTGAGE30US", "series_name": "the 30-year fixed rate mortgage average in the US"}, - { - "id": "NASDAQ100", - "series_name": "the NASDAQ 100 Index, which represents the daily index value at market close,", - }, - { - "id": "NASDAQCOM", - "series_name": "the NASDAQ Composite Index, which represents the daily index value at market close,", - }, - { - "id": "NDR12MCD", - "series_name": "the US national deposit rate for 12-month certificates of deposit (CDs)", - }, - {"id": "NFCI", "series_name": "the Chicago Fed's National Financial Conditions Index"}, - { - "id": "NFCICREDIT", - "series_name": "the Chicago Fed's National Financial Conditions Credit Subindex", - }, - { - "id": "NFCILEVERAGE", - "series_name": "the Chicago Fed's National Financial Conditions Leverage Subindex", - }, - { - "id": "NFCIRISK", - "series_name": "the Chicago Fed's National Financial Conditions Risk Subindex", - }, - { - "id": "NIKKEI225", - "series_name": "the Nikkei 225 Stock Average, which represent the daily index value at market close,", - }, - {"id": "OBFR", "series_name": "the Federal Reserve's overnight bank funding rate"}, - {"id": "OBMMIFHA30YF", "series_name": "the 30-year fixed rate FHA mortgage index"}, - {"id": "OBMMIJUMBO30YF", "series_name": "the 30-year fixed rate jumbo mortgage index"}, - {"id": "OBMMIVA30YF", "series_name": "the 30-year fixed rate Veterans Affairs mortgage index"}, - { - "id": "OVXCLS", - "series_name": "the Chicago Board Options Exchange's Crude Oil ETF Volatility Index", - }, - { - "id": "REAINTRATREARAT10Y", - "series_name": "the Federal Reserve Bank of Cleveland's estimate for the 10-year real interest rate", - }, - { - "id": "REAINTRATREARAT1YE", - "series_name": "the Federal Reserve Bank of Cleveland's estimate for the 1-year real interest rate", - }, - { - "id": "RESPPANWW", - "series_name": "the total dollar amount of assets held by all US Federal Reserve banks", - }, - { - "id": "RESPPLLOPNWW", - "series_name": "the total weekly remittance of earnings by the Federal Reserve to the US Treasury", - }, - { - "id": "RIFSPPFAAD90NB", - "series_name": "the 90-day AA Financial Commercial Paper Interest Rate", - }, - { - "id": "RPONTSYD", - "series_name": "the aggregated daily value of US Treasury securities repurchased overnight by the Federal Reserve in temporary open market operations ", - }, - { - "id": "RRPONTSYAWARD", - "series_name": "the award rate of US Treasury securities sold by the Federal Reserve in overnight temporary open market operations", - }, - { - "id": "RRPONTSYD", - "series_name": "the aggregated daily value of US Treasury securities sold by the Federal Reserve in temporary open market operations", - }, - { - "id": "RRPONTTLD", - "series_name": "the aggregated daily value of securities sold by the Federal Reserve in temporary open market operations", - }, - { - "id": "SNDR", - "series_name": "the aggregated value of US national average interest rates for savings accounts", - }, - {"id": "SOFR", "series_name": "the Federal Reserve's Secured Overnight Financing Rate"}, - { - "id": "SOFR180DAYAVG", - "series_name": "the 180-day average of the Federal Reserve's Secured Overnight Financing Rate", - }, - { - "id": "SOFR30DAYAVG", - "series_name": "the 30-day average of the Federal Reserve's Secured Overnight Financing Rate", - }, - { - "id": "SOFR90DAYAVG", - "series_name": "the 90-day average of the Federal Reserve's Secured Overnight Financing Rate", - }, - { - "id": "SOFRINDEX", - "series_name": "the Federal Reserve's SOFR (Secured Overnight Financing Rate) Index", - }, - { - "id": "SP500", - "series_name": "the S&P 500, which represents the daily index value at market close", - }, - {"id": "STLFSI4", "series_name": "the St. Louis Fed Financial Stress Index"}, - { - "id": "SWPT", - "series_name": "the weekly value of central bank liquidity swaps held by the Federal Reserve", - }, - { - "id": "T10Y2Y", - "series_name": "the yield spread between 10-year and 2-year US Treasury bonds", - }, - { - "id": "T10Y3M", - "series_name": "the yield spread between 10-year and 3-month US Treasury bonds", - }, - { - "id": "T10YFF", - "series_name": "the yield spread between the 10-year US Treasury bond and the Effective Federal Funds Rate (interest rate)", - }, - {"id": "T10YIE", "series_name": "the US' 10-year breakeven inflation rate"}, - {"id": "T5YIE", "series_name": "the US' 5-year breakeven inflation rate"}, - {"id": "T5YIFR", "series_name": "the US' 5-year forward inflation expectation rate"}, - {"id": "THREEFYTP10", "series_name": "the term premium on a 10-year zero-coupon bond"}, - { - "id": "TLAACBW027SBOG", - "series_name": "the total dollar amount of assets held by all US commercial banks", - }, - { - "id": "TMBACBW027SBOG", - "series_name": "the total dollar amount of mortgage-backed securities held by all US commercial banks", - }, - { - "id": "TOTBKCR", - "series_name": "the total dollar amount of bank credit held by all US commercial banks", - }, - { - "id": "TOTCI", - "series_name": "the total dollar amount representing all commercial and industrial loans made by commercial banks in the US", - }, - { - "id": "TOTLL", - "series_name": "the total dollar amount representing all loans and leases in bank credit made by commercial banks in the US", - }, - { - "id": "TREAST", - "series_name": "the total value of US Treasury securities held by the Federal Reserve", - }, - { - "id": "USEPUINDXD", - "series_name": "the Economic Policy Uncertainty Index for the US", - }, - {"id": "VIXCLS", "series_name": "the Chicago Board Options Exchange's Volatility Index"}, - { - "id": "VXVCLS", - "series_name": "the Chicago Board Options Exchange's S&P 500 3-Month Volatility Index", - }, - { - "id": "WALCL", - "series_name": "the total dollar amount of assets held by all US Federal Reserve banks", - }, - { - "id": "WDTGAL", - "series_name": "the total dollar amount of deposits in the US Treasury's general accounts of Federal Reserve Banks, other than reserve balances,", - }, - {"id": "WEI", "series_name": "the Weekly Economic Index (Lewis-Mertens-Stock)"}, - { - "id": "WGS10YR", - "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis,", - }, - { - "id": "WGS1YR", - "series_name": "the market yield on US treasury securities at 1-year constant maturity, quoted on an investment basis,", - }, - { - "id": "WLCFLL", - "series_name": "the weekly dollar amount of loans made by the Federal Reserve under its liquidity and credit facilities", - }, - { - "id": "WLCFLPCL", - "series_name": "the weekly dollar amount of loans made under the primary credit lending program by the Federal Reserve", - }, - { - "id": "WLODLL", - "series_name": "the weekly dollar amount of balances in the accounts of depository institutions in the Federal Reserve Banks", - }, - { - "id": "WLRRAL", - "series_name": "the weekly dollar amount associated with Federal Reserve reverse repurchase agreements", - }, - {"id": "WM1NS", "series_name": "USD money supply as measured by M1"}, - {"id": "WM2NS", "series_name": "USD money supply as measured by M2"}, - { - "id": "WORAL", - "series_name": "the weekly dollar amount associated with Federal Reserve repurchase agreements", - }, - { - "id": "WRBWFRBL", - "series_name": "the total dollar amount of reserve balances held with Federal Reverse Banks", - }, - { - "id": "WRESBAL", - "series_name": "the weekly average of reserve balances held with Federal Reserve Banks", - }, - { - "id": "WRMFNS", - "series_name": "Retail Money Market Funds, a component of M2, a measure of USD money supply,", - }, - { - "id": "WSHOMCB", - "series_name": "the total dollar amount of mortgage-backed securities held by the US Federal Reserve Banks", - }, - { - "id": "WSHOSHO", - "series_name": "the total dollar amount of securities held by US Federal Reserve Banks", - }, - { - "id": "WTREGEN", - "series_name": "the weekly average of deposits other than reserve balances held in the US treasury's general accounts with Federal Reserve Banks", - }, - { - "id": "LNU01075379", - "series_name": "the number of US civilians employed or available for employment with no disability and 65 years old and older", - }, - { - "id": "CGRAL16O", - "series_name": "the number of US civilians employed or available for employment with a Bachelor's Degree or higher and 16 years old and older", - }, - { - "id": "ADEGL16O", - "series_name": "the number of US civilians 16 years old and older employed or available for employment with an Associate Degree", - }, - { - "id": "LNU01075600", - "series_name": "the number of US civilians 65 years old and older employed or available for employment with a disability", - }, - { - "id": "LNU01073397", - "series_name": "the number of foreign born, female US civilians employed or available for employment", - }, - { - "id": "LNU01073396", - "series_name": "the number of foreign born, male US civilians employed or available for employment", - }, - { - "id": "LNU01073415", - "series_name": "the number of native born, female US civilians employed or available for employment", - }, - { - "id": "LNU01074597", - "series_name": "the number of US civilians 16 years old and older employed or available for employment with a disability", - }, - { - "id": "CLF16OV", - "series_name": "the number of US civilians employed or available for employment", - }, - { - "id": "LNU01073395", - "series_name": "the number of foreign born US civilians employed or available for employment", - }, - { - "id": "LNS11000060", - "series_name": "the number of US civilians between 25 and 54 years of age that are employed or available for employment", - }, - { - "id": "LNU01076960", - "series_name": "the number of female US civilians employed or available for employment with a disability and between 16 and 64 years of age", - }, - { - "id": "LNU01073413", - "series_name": "the number of foreign born US civilians employed or available for employment", - }, - { - "id": "LNS11024230", - "series_name": "the number of US civilians aged 55 years and above employed or available for employment", - }, - { - "id": "LNS11000002", - "series_name": "the number of female US civilians employed or available for employment", - }, - { - "id": "TOTLL65O", - "series_name": "the number of US civilians aged 65 years and above employed or available for employment", - }, - { - "id": "LNS11000001", - "series_name": "the number of male US civilians employed or available for employment", - }, - { - "id": "LNU01076955", - "series_name": "the number of male US civilians employed or available for employment with a disability and between 16 and 64 years of age", - }, - { - "id": "LNS11000009", - "series_name": "the number of Hispanic or Latino US civilians employed or available for employment", - }, - { - "id": "LNS11000003", - "series_name": "the number of White US civilians employed or available for employment", - }, - { - "id": "LNS11000006", - "series_name": "the number of Black or African American US civilians employed or available for employment", - }, - { - "id": "TOTLL3544", - "series_name": "the number of US civilians employed or available for employment between 35 and 44 years of age", - }, - { - "id": "TOTLL5564", - "series_name": "the number of US civilians employed or available for employment between 55 and 64 years of age", - }, - { - "id": "TOTLL2534", - "series_name": "the number of US civilians employed or available for employment between 25 and 34 years of age", - }, - { - "id": "LNS11000036", - "series_name": "the number of US civilians employed or available for employment between 20 and 24 years of age", - }, - { - "id": "LNS11000012", - "series_name": "the number of US civilians employed or available for employment between 16 and 19 years of age", - }, - { - "id": "LNU01032183", - "series_name": "the number of Asian US civilians employed or available for employment", - }, - { - "id": "LNU01375600", - "series_name": "the labor force participation rate among US civilians 65 years and older with a disability", - }, - { - "id": "LNU01373414", - "series_name": "the labor force participation rate among native born, male US civilians", - }, - { - "id": "LNU01373396", - "series_name": "the labor force participation rate among foreign born, male US civilians", - }, - { - "id": "LNU01373415", - "series_name": "the labor force participation rate among native born, female US civilians", - }, - { - "id": "LNU01373397", - "series_name": "the labor force participation rate among foreign born, female US civilians", - }, - { - "id": "LNU01300003", - "series_name": "the labor force participation rate among White US civilians", - }, - { - "id": "LNU01373395", - "series_name": "the labor force participation rate among foreign born US civilians", - }, - { - "id": "LNU01300009", - "series_name": "the labor force participation rate among Hispanic or Latino US civilians", - }, - { - "id": "LNU01373413", - "series_name": "the labor force participation rate among native born US civilians", - }, - { - "id": "LNU01332183", - "series_name": "the labor force participation rate among Asian US civilians", - }, - { - "id": "CIVPART", - "series_name": "the labor force participation rate among US civilians", - }, - { - "id": "LNS11300060", - "series_name": "the labor force participation rate among US civilians between 25 and 54 years of age", - }, - { - "id": "LNU01300002", - "series_name": "the labor force participation rate among female US civilians", - }, - { - "id": "LNU01300001", - "series_name": "the labor force participation rate among male US civilians", - }, - { - "id": "LNS11324230", - "series_name": "the labor force participation rate among US civilians 55 years old and older", - }, - { - "id": "LNU01300012", - "series_name": "the labor force participation rate among US civilians between 16 and 19 years of age", - }, - { - "id": "LNU01300006", - "series_name": "the labor force participation rate among Black or African American US civilians", - }, - { - "id": "LNS11300036", - "series_name": "the labor force participation rate among US civilians between 20 and 24 years of age", - }, - { - "id": "LNU01375379", - "series_name": "the labor force participation rate among US civilians 65 years old and older with no disability", - }, - { - "id": "LNU01374597", - "series_name": "the labor force participation rate among US civilians 16 years old and older with a disability", - }, - { - "id": "LNU01327662", - "series_name": "the labor force participation rate among US civilians 25 years old and older with a Bachelor's Degree", - }, - { - "id": "LNS12600000", - "series_name": "the number of employed US civilians who usually work part time", - }, - { - "id": "LNS12500000", - "series_name": "the number of employed US civilians who usually work full time", - }, - { - "id": "LNU02075600", - "series_name": "the number of employed US civilians 65 years old and older with a disability", - }, - { - "id": "LNU02074597", - "series_name": "the number of employed US civilians 16 years old and older with a disability", - }, - { - "id": "LNU02074593", - "series_name": "the number of employed US civilians 16 years old and older with no disability", - }, - { - "id": "LNU02075379", - "series_name": "the number of employed US civilians 65 years old and older with no disability", - }, - { - "id": "CE16OV", - "series_name": "the number of employed US civilians", - }, - { - "id": "LNU02000086", - "series_name": "the number of employed US civilians between 16 and 17 years of age", - }, - { - "id": "LNS12000012", - "series_name": "the number of employed US civilians between 16 and 19 years of age", - }, - { - "id": "LNS12000088", - "series_name": "the number of employed US civilians between 18 and 19 years of age", - }, - { - "id": "LNS12000036", - "series_name": "the number of employed US civilians between 20 and 24 years of age", - }, - { - "id": "LNS12000024", - "series_name": "the number of employed US civilians 20 years old and older", - }, - { - "id": "LNS12000089", - "series_name": "the number of employed US civilians between 25 and 34 years of age", - }, - { - "id": "LNS12000060", - "series_name": "the number of employed US civilians between 25 and 54 years of age", - }, - { - "id": "LNS12000048", - "series_name": "the number of employed US civilians 25 years old and older", - }, - { - "id": "LNS12000091", - "series_name": "the number of employed US civilians between 35 and 44 years of age", - }, - { - "id": "LNS12000093", - "series_name": "the number of employed US civilians between 45 and 54 years of age", - }, - { - "id": "LNS12024230", - "series_name": "the number of employed US civilians 55 years old and older", - }, - { - "id": "LNS12034560", - "series_name": "the number of US civilians employed in agriculture and related industries", - }, - { - "id": "LNS12027714", - "series_name": "the number of self-employed, unincorporated US civilians", - }, - { - "id": "LNU02032183", - "series_name": "the number of employed Asian US civilians", - }, - { - "id": "LNS12027662", - "series_name": "the number of employed US civilians 25 years old and older with a Bachelor's Degree and higher", - }, - { - "id": "LNS12000006", - "series_name": "the number of employed Black or African American US civilians", - }, - { - "id": "LNU02032210", - "series_name": "the number of US civilians employed in construction and extraction occupations", - }, - { - "id": "LNU02032209", - "series_name": "the number of US civilians employed in farming, fishing and forestry occupations", - }, - { - "id": "LNU02073395", - "series_name": "the number of employed foreign born US civilians", - }, - { - "id": "LNS12000009", - "series_name": "the number of employed Hispanic or Latino US civilians", - }, - { - "id": "LNU02032211", - "series_name": "the number of US civilians employed in installation, maintenance and repair occupations", - }, - { - "id": "LNU02032202", - "series_name": "the number of US civilians employed in management, business, and financial operations occupations", - }, - { - "id": "LNU02032201", - "series_name": "the number of US civilians employed in management, professional, and related occupations", - }, - { - "id": "LNS12000001", - "series_name": "the number of employed male US civilians", - }, - { - "id": "LNU02073413", - "series_name": "the number of employed native born US civilians", - }, - { - "id": "LNU02032208", - "series_name": "the number of US civilians employed in natural resources, construction, and maintenance occupations", - }, - { - "id": "LNS12035019", - "series_name": "the number of US civilians employed in nonagricultural industries", - }, - { - "id": "LNU02032207", - "series_name": "the number of US civilians employed in office and administrative support occupations", - }, - { - "id": "LNS12032197", - "series_name": "the number of US civilians employed part-time for economic reasons in nonagricultural industries", - }, - { - "id": "LNS12032199", - "series_name": "the number of US civilians employed part-time for economic reasons in nonagricultural industries, who could only find part-time work", - }, - { - "id": "LNS12032200", - "series_name": "the number of employed US civilians employed part-time for noneconomic reasons in nonagricultural industries", - }, - { - "id": "LNU02032213", - "series_name": "the number of US civilians employed in production occupations", - }, - { - "id": "LNU02032212", - "series_name": "the number of US civilians employed in production, transportation and material moving occupations", - }, - { - "id": "LNU02032203", - "series_name": "the number of US civilians employed in professional and related occupations", - }, - { - "id": "LNU02032205", - "series_name": "the number of US civilians employed in sales and office occupations", - }, - { - "id": "LNU02032206", - "series_name": "the number of US civilians employed in sales and related occupations", - }, - { - "id": "LNU02032204", - "series_name": "the number of US civilians employed in service occupations", - }, - { - "id": "LNU02048984", - "series_name": "the number of incorporated self-employed US civilians", - }, - { - "id": "LNS12000003", - "series_name": "the number of employed White US civilians", - }, - { - "id": "LNS12000002", - "series_name": "the number of employed female US civilians", - }, - { - "id": "LNU02374597", - "series_name": "the employment-population ratio for US civilians 16 years and older with a disability", - }, - { - "id": "LNU02375600", - "series_name": "the employment-population ratio for US civilians 65 years and older with a disability", - }, - { - "id": "LNU02374593", - "series_name": "the employment-population ratio for US civilians 16 years and older with no disability", - }, - { - "id": "LNU02375379", - "series_name": "the employment-population ratio for US civilians 65 years and older with no disability", - }, - { - "id": "LNS12300002", - "series_name": "the employment-population ratio for female US civilians", - }, - { - "id": "LNS12327689", - "series_name": "the employment-population ratio for US civilians 25 years and older with some college or associate degree", - }, - { - "id": "LNS12327660", - "series_name": "the employment-population ratio for US civilians 25 years and older with a high school diploma", - }, - { - "id": "LNS12327659", - "series_name": "the employment-population ratio for US civilians 25 years and older with less than a high school diploma", - }, - { - "id": "EMRATIO", - "series_name": "the employment-population ratio for US civilians", - }, - { - "id": "LNS12300012", - "series_name": "the employment-population ratio for US civilians between 16 and 19 years of age", - }, - { - "id": "LNS12300060", - "series_name": "the employment-population ratio for US civilians between 25 and 54 years of age", - }, - { - "id": "LNU02332183", - "series_name": "the employment-population ratio for Asian US civilians", - }, - { - "id": "LNS12327662", - "series_name": "the employment-population ratio for US civilians 25 years and older with a Bachelor's degree and higher", - }, - { - "id": "LNS12300006", - "series_name": "the employment-population ratio for Black or African American US civilians", - }, - { - "id": "LNU02373395", - "series_name": "the employment-population ratio for foreign born US civilians", - }, - { - "id": "LNS12300009", - "series_name": "the employment-population ratio for Hispanic or Latino US civilians", - }, - { - "id": "LNS12300001", - "series_name": "the employment-population ratio for male US civilians", - }, - { - "id": "LNU02373413", - "series_name": "the employment-population ratio for native born US civilians", - }, - { - "id": "LNS12300003", - "series_name": "the employment-population ratio for White US civilians", - }, - { - "id": "LNU03074597", - "series_name": "number of unemployed US civilians 16 years and older with a disability", - }, - { - "id": "LNU03075600", - "series_name": "number of unemployed US civilians 65 years and older with a disability", - }, - { - "id": "LNU03074593", - "series_name": "number of unemployed US civilians 16 years and older with no disability", - }, - { - "id": "LNU03075379", - "series_name": "number of unemployed US civilians 65 years and older with no disability", - }, - { - "id": "UNEMPLOY", - "series_name": "number of unemployed US civilians", - }, - { - "id": "LNS13000012", - "series_name": "number of unemployed US civilians between 16 and 19 years of age", - }, - { - "id": "LNS13000036", - "series_name": "number of unemployed US civilians between 20 and 24 years of age", - }, - { - "id": "LNS13000089", - "series_name": "number of unemployed US civilians between 25 and 34 years of age", - }, - { - "id": "TOTLU2564", - "series_name": "number of unemployed US civilians between 25 and 64 years of age", - }, - { - "id": "TOTLU25O", - "series_name": "number of unemployed US civilians 25 years and older", - }, - { - "id": "TOTLU3544", - "series_name": "number of unemployed US civilians between 35 and 44 years of age", - }, - { - "id": "LNS13000093", - "series_name": "number of unemployed US civilians between 45 and 54 years of age", - }, - { - "id": "TOTLU5564", - "series_name": "number of unemployed US civilians between 55 and 64 years of age", - }, - { - "id": "TOTLU65O", - "series_name": "number of unemployed US civilians 65 years and older", - }, - { - "id": "LNU03032183", - "series_name": "number of unemployed Asian US civilians", - }, - { - "id": "ADEGU16O", - "series_name": "number of unemployed US civilians 16 years and older with an associate degree", - }, - { - "id": "ADAPU16O", - "series_name": "number of unemployed US civilians 16 years and older with an associate degree (academic program)", - }, - { - "id": "ADOPU16O", - "series_name": "number of unemployed US civilians 16 years and older with an associate degree (occupational program)", - }, - { - "id": "CGRAU16O", - "series_name": "number of unemployed US civilians 16 years and older with a Bachelor's degree or higher", - }, - { - "id": "LNS13000006", - "series_name": "number of unemployed Black or African American US civilians", - }, - { - "id": "CGBDU16O", - "series_name": "number of unemployed US civilians 16 years and older with a doctoral degree", - }, - { - "id": "CGMDU16O", - "series_name": "number of unemployed US civilians 16 years and older with a Master's degree", - }, - { - "id": "LNU03073395", - "series_name": "number of unemployed foreign born US civilians", - }, - { - "id": "HSGSU16O", - "series_name": "number of unemployed US civilians 16 years and older with a high school diploma", - }, - { - "id": "LNS13000009", - "series_name": "number of unemployed Hispanic or Latino US civilians", - }, - { - "id": "LNU03023705", - "series_name": "number of unemployed US civilians who left (as opposed to lost) their job", - }, - { - "id": "LNU03023621", - "series_name": "number of unemployed US civilians who lost (as opposed to left) their job", - }, - { - "id": "LNS13025699", - "series_name": "number of unemployed US civilians who lost their job not on layoff", - }, - { - "id": "LNS13023653", - "series_name": "number of unemployed US civilians who lost their job on layoff", - }, - { - "id": "LHSDU16O", - "series_name": "number of unemployed US civilians 16 years and older with less than a high school diploma", - }, - { - "id": "LNS13100000", - "series_name": "number of unemployed US civilians looking for full-time work", - }, - { - "id": "LNS13200000", - "series_name": "number of unemployed US civilians looking for part-time work", - }, - { - "id": "LNS13000001", - "series_name": "number of unemployed male US civilians", - }, - { - "id": "LNU03073413", - "series_name": "number of unemployed native born US civilians", - }, - { - "id": "LNU03023569", - "series_name": "number of unemployed US civilians who are new entrants", - }, - { - "id": "LNS13026638", - "series_name": "number of permanently unemployed US civilians", - }, - { - "id": "LNS13026637", - "series_name": "number of unemployed US civilians who completed temporary jobs", - }, - { - "id": "SCADU16O", - "series_name": "number of unemployed US civilians 16 years and older with some college or associate degree", - }, - { - "id": "LNS13000003", - "series_name": "number of unemployed White US civilians", - }, - { - "id": "LNS13000002", - "series_name": "number of unemployed female US civilians", - }, - { - "id": "LNU03000313", - "series_name": "number of unemployed female US civilians who maintain families", - }, - { - "id": "LNU04000006", - "series_name": "the unemployement rate for Black or African American US civilians", - }, - { - "id": "CGBD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with a Bachelor's degree", - }, - { - "id": "CGDD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with a doctoral degree", - }, - { - "id": "CGMD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with a Master's degree", - }, - { - "id": "CGPD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with a professional degree", - }, - { - "id": "LNU04032240", - "series_name": "the unemployement rate for US private wage and salary workers in education and health services", - }, - { - "id": "LNU04032233", - "series_name": "the unemployement rate for US wage and salary workers in the durable goods industry", - }, - { - "id": "LNU04032231", - "series_name": "the unemployement rate for US private wage and salary workers in the construction industry", - }, - { - "id": "LNU04032224", - "series_name": "the unemployement rate for US civilians in construction and extraction occupations", - }, - { - "id": "LNU04032223", - "series_name": "the unemployement rate for US civilians in farming, fishing, and forestry occupations", - }, - { - "id": "LNU04032238", - "series_name": "the unemployement rate for US private wage and salary workers in the financial activities industry", - }, - { - "id": "LNU04073395", - "series_name": "the unemployement rate for foreign born US civilians", - }, - { - "id": "LNS14100000", - "series_name": "the unemployement rate for US full-time workers", - }, - { - "id": "HSGS16O", - "series_name": "the unemployement rate for US civilians 16 years and older with a high school diploma", - }, - { - "id": "LNU04000009", - "series_name": "the unemployement rate for Hispanic or Latino US civilians", - }, - { - "id": "LNU04032237", - "series_name": "the unemployement rate for US private wage and salary workers in the information industry", - }, - { - "id": "LNU04032225", - "series_name": "the unemployement rate for US civilians in installation, maintenance, and repair occupations", - }, - { - "id": "LNS14023705", - "series_name": "the unemployement rate for US civilians who left (as opposed to lost) their job", - }, - { - "id": "LNU04032241", - "series_name": "the unemployement rate for US private wage and salary workers in leisure and hospitality", - }, - { - "id": "LHSD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with less than a high school diploma", - }, - { - "id": "LNU04032232", - "series_name": "the unemployement rate for US private wage and salary workers in the manufacturing industry", - }, - { - "id": "LNU04032215", - "series_name": "the unemployement rate for US civilians in management, professional, and related occupations", - }, - { - "id": "LNU04032216", - "series_name": "the unemployement rate for US civilians in management, business, and financial operations occupations", - }, - { - "id": "LNS14000001", - "series_name": "the unemployement rate for male US civilians", - }, - { - "id": "LNU04073413", - "series_name": "the unemployement rate for native born US civilians", - }, - { - "id": "LNS14023569", - "series_name": "the unemployement rate for US civilians who are new entrants", - }, - { - "id": "LNU04032229", - "series_name": "the unemployement rate for US private wage and salary workers in nonagriculture occupations", - }, - { - "id": "LNU04032234", - "series_name": "the unemployement rate for US private wage and salary workers in the non durable goods industry", - }, - { - "id": "LNU04032221", - "series_name": "the unemployement rate for US civilians in office and administrative support occupations", - }, - { - "id": "LNS14200000", - "series_name": "the unemployement rate for US part-time workers", - }, - { - "id": "LNU04032227", - "series_name": "the unemployement rate for US civilians in production occupations", - }, - { - "id": "LNU04032239", - "series_name": "the unemployement rate for US private wage and salary workers in the professional and business services industry", - }, - { - "id": "LNU04032226", - "series_name": "the unemployement rate for US civilians in production, transportation and material moving occupations", - }, - { - "id": "LNU04032217", - "series_name": "the unemployement rate for US civilians in professional and related occupations", - }, - { - "id": "LNS14023557", - "series_name": "the unemployement rate for US reentrants to labor force", - }, - { - "id": "LNU04032219", - "series_name": "the unemployement rate for US civilians in the sales and office occupations", - }, - { - "id": "LNU04032218", - "series_name": "the unemployement rate for US civilians in service occupations", - }, - { - "id": "SCAD16O", - "series_name": "the unemployement rate for US civilians 16 years and older with some college or associate degree", - }, - { - "id": "LNU04032228", - "series_name": "the unemployement rate for US civilians in transportation and material moving occupations", - }, - { - "id": "LNU04032236", - "series_name": "the unemployement rate for US wage and salary workers in transportation and utilities industries", - }, - { - "id": "LNU04000003", - "series_name": "the unemployement rate for White US civilians", - }, - { - "id": "LNU04075600", - "series_name": "the unemployement rate for US civilians 65 years and older with a disability", - }, - { - "id": "LNU04074597", - "series_name": "the unemployement rate for US civilians 16 years and older with a disability", - }, - { - "id": "LNU04074593", - "series_name": "the unemployement rate for US civilians 16 years and older with no disability", - }, - { - "id": "LNU04075379", - "series_name": "the unemployement rate for US civilians 65 years and older with no disability", - }, - { - "id": "LNS14000002", - "series_name": "the unemployement rate for female US civilians", - }, - { - "id": "LNU04000313", - "series_name": "the unemployement rate for female US civilians who maintain families", - }, - { - "id": "UNRATE", - "series_name": "the unemployement rate for US civilian labor force", - }, - { - "id": "LNU04000012", - "series_name": "the unemployement rate for US civilians between 16 and 19 years of age", - }, - { - "id": "LNS14000036", - "series_name": "the unemployement rate for US civilians between 20 and 24 years of age", - }, - { - "id": "LNS14000089", - "series_name": "the unemployement rate for US civilians between 25 and 34 years of age", - }, - { - "id": "LNS14000060", - "series_name": "the unemployement rate for US civilians between 25 and 54 years of age", - }, - { - "id": "TOTL2564", - "series_name": "the unemployement rate for US civilians between 25 and 64 years of age", - }, - { - "id": "LNS14000091", - "series_name": "the unemployement rate for US civilians between 35 and 44 years of age", - }, - { - "id": "LNS14000093", - "series_name": "the unemployement rate for US civilians between 45 sand 54 years of age", - }, - { - "id": "LNU04000095", - "series_name": "the unemployement rate for US civilians between 55 and 64 years of age", - }, - { - "id": "LNU04000097", - "series_name": "the unemployement rate for US civilians 65 years and older", - }, - { - "id": "LNS14032183", - "series_name": "the unemployement rate for Asian US civilians", - }, - { - "id": "ADEG16O", - "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree", - }, - { - "id": "ADAP16O", - "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree (academic program)", - }, - { - "id": "ADOP16O", - "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree (occupational program)", - }, - { - "id": "LNU05000003", - "series_name": "the number of White US civilians not in the labor force", - }, - { - "id": "LNU05074597", - "series_name": "the number of US civilians 16 years and older with a disability who are not in the labor force", - }, - { - "id": "LNU05075600", - "series_name": "the number of US civilians 65 years and older with a disability who are not in the labor force", - }, - { - "id": "LNU05074593", - "series_name": "the number of US civilians 16 years and older with no disability who are not in the labor force", - }, - { - "id": "LNU05075379", - "series_name": "the number of US civilians 65 years and older with no disability who are not in the labor force", - }, - { - "id": "LNU05000002", - "series_name": "the number of female US civilians not in the labor force", - }, - { - "id": "LNU05000001", - "series_name": "the number of male US civilians not in the labor force", - }, - { - "id": "LNU05026640", - "series_name": "the number of male US civilians not in the labor force who want a job now", - }, - { - "id": "LNU05026641", - "series_name": "the number of female US civilians not in the labor force who want a job now", - }, - { - "id": "LNU05000009", - "series_name": "the number of Hispanic or Latino US civilians not in the labor force", - }, - { - "id": "LNU05000006", - "series_name": "the number of Black or African American US civilians not in the labor force", - }, - { - "id": "LNU05000012", - "series_name": "the number of US civilians between 16 and 19 not in the labor force", - }, - { - "id": "LNU05000000", - "series_name": "the number of US civilians not in the labor force", - }, - { - "id": "LNU05073395", - "series_name": "the number of foreign born US civilians not in the labor force", - }, - { - "id": "LNU05032183", - "series_name": "the number of Asian US civilians not in the labor force", - }, - { - "id": "LNU05073413", - "series_name": "the number of native born US civilians not in the labor force", - }, - { - "id": "LNS11300012", - "series_name": "the labor force participation rate of US civilians between 16 and 19 years of age", - }, - { - "id": "LNS11300006", - "series_name": "the labor force participation rate of Black or African American US civilians", - }, - { - "id": "LNS11327662", - "series_name": "the labor force participation rate of US civilians 25 years and older with a Bachelor's degree and higher", - }, - { - "id": "LNS11300003", - "series_name": "the labor force participation rate of White US civilians", - }, - { - "id": "LNS11327660", - "series_name": "the labor force participation rate of US civilians 25 years and older with a high school diploma", - }, - { - "id": "LNS11300009", - "series_name": "the labor force participation rate of Hispanic or Latino US civilians", - }, - { - "id": "LNS11327689", - "series_name": "the labor force participation rate of US civilians 25 years and older with some college or associate degree", - }, - { - "id": "LNS11300002", - "series_name": "the labor force participation rate of female US civilians", - }, - { - "id": "LNS11300001", - "series_name": "the labor force participation rate of male US civilians", - }, - { - "id": "LNS12026620", - "series_name": "the percentage of employed US civilians who have more than one job", - }, - { - "id": "LNU02026631", - "series_name": "the number of US civilians who have more than one full-time job", - }, - { - "id": "LNU02026625", - "series_name": "the number of US civilians who have one full-time and at least one part-time job", - }, - { - "id": "LNS12026619", - "series_name": "the number of US civilians who have more than one job", - }, - { - "id": "LNU02026628", - "series_name": "the numer of US civilians who have at least two part-time jobs", - }, - { - "id": "LNU02026623", - "series_name": "the number of female US civilians who have more than one job", - }, - { - "id": "LNU02026624", - "series_name": "the percentage of employed, female US civilians who have more than one job", - }, - { - "id": "LNU02026622", - "series_name": "the percentage of employed, male US civilians who have more than one job", - }, - { - "id": "LNU02026621", - "series_name": "the number of male US civilians who have more than one job", - }, - { - "id": "UEMPMEAN", - "series_name": "the average number of weeks that US civilians have been unemployed", - }, - { - "id": "LNU03008276", - "series_name": "the median number of weeks that US civilians have been unemployed", - }, - { - "id": "LNU03008636", - "series_name": "the number of US civilians who have been unemployed for 27 weeks or more", - }, - { - "id": "LNU03008396", - "series_name": "the number of US civilians who have been unemployed for 5 weeks or less", - }, - { - "id": "LNS13025703", - "series_name": "the percentage of unemployed US civilians who have been unemployed for 27 weeks or more", - }, - { - "id": "LNU03008756", - "series_name": "the nubmer of US civilians who have been unemployed for 5 to 14 weeks", - }, - { - "id": "LNS13008397", - "series_name": "the percentage of unemployed US civilians who have been unemployed for less than 5 weeks", - }, - { - "id": "LNS13023622", - "series_name": "the percentage of unemployed US civilians who have lost (as opposed to left) their job", - }, - { - "id": "LNS13023706", - "series_name": "the percentage of unemployed US civilians who have left (as opposed to lost) their job", - }, - { - "id": "LNS13023654", - "series_name": "the percentage of unemployed US civilians who have lost their job on layoff", - }, - { - "id": "LNS13026511", - "series_name": "the percentage of unemployed US civilians who have not lost their job on layoff", - }, - { - "id": "LNS13023558", - "series_name": "the percentage of unemployed US civilians who are reentrants", - }, - { - "id": "LNS13023570", - "series_name": "the percentage of unemployed US civilians who are new entrants", - }, - { - "id": "LNS17800000", - "series_name": "the number of US civilians who went from 'employed' to 'not in labor force'", - }, - { - "id": "LNS17900000", - "series_name": "the number of US civilians who went from 'unemployed' to 'not in labor force'", - }, - { - "id": "LNS17000000", - "series_name": "the number of US civilians who remain employed", - }, - { - "id": "LNS17400000", - "series_name": "the number of US civilians who went from 'employed' to 'unemployed'", - }, - { - "id": "LNS17100000", - "series_name": "the number of US civilians who went from 'unemployed' to 'employed'", - }, - { - "id": "LNS17200000", - "series_name": "the number of US civilians who went from 'not in labor force' to 'employed'", - }, - { - "id": "LNS17600000", - "series_name": "the number of US civilians who went from 'not in labor force' to 'unemployed'", - }, - { - "id": "LNS17500000", - "series_name": "the number of US civilians who remained unemployed", - }, - { - "id": "LNS18000000", - "series_name": "the number of US civilians who remained 'not in labor force'", - }, - { - "id": "AWHAETP", - "series_name": "the average weekly hours of US employees in the private sector", - }, - { - "id": "CES0600000010", - "series_name": "the total number of female US employees in goods-producing businesses", - }, - { - "id": "CES1021100001", - "series_name": "the total number of US employees in oil and gas extraction businesses", - }, - { - "id": "USMINE", - "series_name": "the total number of US employees in mining and logging businesses", - }, - { - "id": "AWHAEMAL", - "series_name": "the average weekly hours of US employees in mining and logging businesses", - }, - { - "id": "CEU1000000011", - "series_name": "the average weekly earnings of US employees in mining and logging businesses", - }, - { - "id": "CEU1000000010", - "series_name": "the total number of female US employees in mining and logging businesses", - }, - { - "id": "AWHAEGP", - "series_name": "the average weekly hours of US employees in goods-producing businesses", - }, - { - "id": "AWHAECON", - "series_name": "the average weekly hours of US employees in construction businesses", - }, - { - "id": "CES2000000039", - "series_name": "the female US employees-to-all US employees ratio in construction businesses", - }, - { - "id": "MANEMP", - "series_name": "the number of US employees in manufacturing", - }, - { - "id": "CES3000000010", - "series_name": "the number of female US employees in manufacturing", - }, - { - "id": "AWHAEDG", - "series_name": "the average weekly hours of US employees in durable goods businesses", - }, - { - "id": "DMANEMP", - "series_name": "the total number of US employees in durable goods businesses", - }, - { - "id": "CES3133400001", - "series_name": "the total number of US employees in computer and electronic product manufacturing", - }, - { - "id": "CES3133440001", - "series_name": "the total number of US employees in semiconductor and other electronic component manufacturing", - }, - { - "id": "CES3133300001", - "series_name": "the total number of US employees in machinery manufacturing", - }, - { - "id": "CES3132100001", - "series_name": "the total number of US employees in wood product manufacturing", - }, - { - "id": "CES3133100001", - "series_name": "the total number of US employees in primary metal manufacturing", - }, - { - "id": "CES3133660001", - "series_name": "the total number of US employees in ship and boat building businesses", - }, - { - "id": "CEU3100000004", - "series_name": "the average weekly overtime hours of US employees in durable goods businesses", - }, - { - "id": "CES3133420001", - "series_name": "the total number of US employees in communications and equipment manufacturing", - }, - { - "id": "CES3100000010", - "series_name": "the total number of female US employees in durable goods businesses", - }, - { - "id": "NDMANEMP", - "series_name": "the total number of US employees in nondurable goods businesses", - }, - { - "id": "CES3231100001", - "series_name": "the total number of US employees in food manufacturing", - }, - { - "id": "CES3232500001", - "series_name": "the total number of US employees in chemical manufacturing", - }, - { - "id": "CES3231500001", - "series_name": "the total number of US employees in apparel manufacturing", - }, - { - "id": "CES3231300001", - "series_name": "the total number of US employees in textile mills", - }, - { - "id": "CES3232600001", - "series_name": "the total number of US employees in plastics and rubber products manufacturing", - }, - { - "id": "CES3232400001", - "series_name": "the total number of US employees in petroleum and coal products manufacturing", - }, - { - "id": "AWHAENDG", - "series_name": "the average weekly hours of US employees in nondurable goods businesses", - }, -] +fred_questions = _META["questions"] diff --git a/src/questions/fred/fetch/Makefile b/src/orchestration/func_fred_fetch/Makefile similarity index 63% rename from src/questions/fred/fetch/Makefile rename to src/orchestration/func_fred_fetch/Makefile index 512d9723..b1350a89 100644 --- a/src/questions/fred/fetch/Makefile +++ b/src/orchestration/func_fred_fetch/Makefile @@ -9,21 +9,25 @@ UPLOAD_DIR = upload .gcloudignore: cp -r $(ROOT_DIR)src/helpers/.gcloudignore . -Procfile: - cp -r $(ROOT_DIR)src/helpers/Procfile . +Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template + sed \ + -e 's/REGION/$(CLOUD_DEPLOY_REGION)/g' \ + -e 's/STACK/google-22-full/g' \ + -e 's/PYTHON_VERSION/python312/g' \ + $< > Dockerfile NUM_CPUS=2 -deploy : main.py .gcloudignore requirements.txt Procfile +deploy : main.py .gcloudignore requirements.txt Dockerfile mkdir -p $(UPLOAD_DIR) cp -r $(ROOT_DIR)utils $(UPLOAD_DIR)/ - cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/ - cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/ - cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ - cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ + cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/helpers + cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/sources mkdir -p $(UPLOAD_DIR)/orchestration cp $(ROOT_DIR)src/orchestration/__init__.py $(UPLOAD_DIR)/orchestration/ - cp $(ROOT_DIR)src/orchestration/_io.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/orchestration/_source_io.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ + cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ cp $^ $(UPLOAD_DIR)/ gcloud run jobs deploy \ func-data-fred-fetch \ @@ -40,4 +44,4 @@ deploy : main.py .gcloudignore requirements.txt Procfile --source $(UPLOAD_DIR) clean : - rm -rf $(UPLOAD_DIR) .gcloudignore Procfile + rm -rf $(UPLOAD_DIR) .gcloudignore Dockerfile diff --git a/src/orchestration/func_fred_fetch/main.py b/src/orchestration/func_fred_fetch/main.py new file mode 100644 index 00000000..ec118ae3 --- /dev/null +++ b/src/orchestration/func_fred_fetch/main.py @@ -0,0 +1,33 @@ +"""FRED fetch entry point.""" + +from __future__ import annotations + +import logging +from typing import Any + +from helpers import data_utils, decorator, keys +from orchestration import _source_io +from sources.fred import FredSource + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +SOURCE = "fred" + + +@decorator.log_runtime +def driver(_: Any) -> None: + """Fetch FRED questions and upload to question bank.""" + source = FredSource() + source.api_key = keys.API_KEY_FRED + + dfq = data_utils.get_data_from_cloud_storage(SOURCE, return_question_data=True) + + dff = source.fetch(dfq=dfq) + + _source_io.write_fetch_output(SOURCE, dff) + logger.info("Done.") + + +if __name__ == "__main__": + driver(None) diff --git a/src/questions/fred/fetch/requirements.txt b/src/orchestration/func_fred_fetch/requirements.txt similarity index 83% rename from src/questions/fred/fetch/requirements.txt rename to src/orchestration/func_fred_fetch/requirements.txt index bf0c8f06..1091a40e 100644 --- a/src/questions/fred/fetch/requirements.txt +++ b/src/orchestration/func_fred_fetch/requirements.txt @@ -1,9 +1,7 @@ google-cloud-storage google-cloud-secret-manager pandas>=2.2.2,<3.0 -tqdm +pandera requests -bs4 backoff -pandera -termcolor +numpy diff --git a/src/questions/fred/update_questions/Makefile b/src/orchestration/func_fred_update/Makefile similarity index 64% rename from src/questions/fred/update_questions/Makefile rename to src/orchestration/func_fred_update/Makefile index cd5e42a1..c06b2c83 100644 --- a/src/questions/fred/update_questions/Makefile +++ b/src/orchestration/func_fred_update/Makefile @@ -9,21 +9,25 @@ UPLOAD_DIR = upload .gcloudignore: cp -r $(ROOT_DIR)src/helpers/.gcloudignore . -Procfile: - cp -r $(ROOT_DIR)src/helpers/Procfile . +Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template + sed \ + -e 's/REGION/$(CLOUD_DEPLOY_REGION)/g' \ + -e 's/STACK/google-22-full/g' \ + -e 's/PYTHON_VERSION/python312/g' \ + $< > Dockerfile NUM_CPUS=2 -deploy : main.py .gcloudignore requirements.txt Procfile +deploy : main.py .gcloudignore requirements.txt Dockerfile mkdir -p $(UPLOAD_DIR) cp -r $(ROOT_DIR)utils $(UPLOAD_DIR)/ - cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/ - cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/ - cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ - cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ + cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/helpers + cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/sources mkdir -p $(UPLOAD_DIR)/orchestration cp $(ROOT_DIR)src/orchestration/__init__.py $(UPLOAD_DIR)/orchestration/ - cp $(ROOT_DIR)src/orchestration/_io.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/orchestration/_source_io.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ + cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ cp $^ $(UPLOAD_DIR)/ gcloud run jobs deploy \ func-data-fred-update-questions \ @@ -40,4 +44,4 @@ deploy : main.py .gcloudignore requirements.txt Procfile --source $(UPLOAD_DIR) clean : - rm -rf $(UPLOAD_DIR) .gcloudignore Procfile + rm -rf $(UPLOAD_DIR) .gcloudignore Dockerfile diff --git a/src/orchestration/func_fred_update/main.py b/src/orchestration/func_fred_update/main.py new file mode 100644 index 00000000..d181ba81 --- /dev/null +++ b/src/orchestration/func_fred_update/main.py @@ -0,0 +1,37 @@ +"""FRED update entry point.""" + +from __future__ import annotations + +import logging +from typing import Any + +from helpers import data_utils, decorator +from orchestration import _source_io +from sources.fred import FredSource + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +SOURCE = "fred" + + +@decorator.log_runtime +def driver(_: Any) -> None: + """Update FRED questions and resolution files.""" + source = FredSource() + + dfq, dff = data_utils.get_data_from_cloud_storage( + SOURCE, return_question_data=True, return_fetch_data=True + ) + + result = source.update(dfq, dff) + + logger.info("Uploading to GCP...") + data_utils.upload_questions(result.dfq, SOURCE) + if result.resolution_files: + _source_io.upload_resolution_files(SOURCE, result.resolution_files) + logger.info("Done.") + + +if __name__ == "__main__": + driver(None) diff --git a/src/questions/fred/update_questions/requirements.txt b/src/orchestration/func_fred_update/requirements.txt similarity index 55% rename from src/questions/fred/update_questions/requirements.txt rename to src/orchestration/func_fred_update/requirements.txt index 2fdeb8d2..961b4524 100644 --- a/src/questions/fred/update_questions/requirements.txt +++ b/src/orchestration/func_fred_update/requirements.txt @@ -1,5 +1,6 @@ google-cloud-storage -google-cloud-secret-manager pandas>=2.2.2,<3.0 pandera -termcolor +requests +backoff +numpy diff --git a/src/questions/fred/fetch/main.py b/src/questions/fred/fetch/main.py deleted file mode 100644 index 3bf58780..00000000 --- a/src/questions/fred/fetch/main.py +++ /dev/null @@ -1,433 +0,0 @@ -"""FRED fetch new questions script.""" - -import json -import logging -import os -import sys -import time -from datetime import datetime, timedelta - -import backoff -import pandas as pd -import requests -from dateutil.relativedelta import relativedelta -from tqdm import tqdm - -sys.path.append(os.path.join(os.path.dirname(__file__), "../../..")) -from helpers import ( # noqa: E402 - constants, - data_utils, - dates, - decorator, - env, - fred, - keys, -) - -sys.path.append(os.path.join(os.path.dirname(__file__), "../../../..")) -from utils import gcp # noqa: E402 - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -SOURCE = "fred" -PARAMS = { - "api_key": keys.API_KEY_FRED, - "file_type": "json", -} - -# FRED throttles each API key at ~2 requests/second (120/minute); exceeding it -# returns HTTP 429. Space requests out to stay safely under that limit. -# See https://fred.stlouisfed.org/docs/api/fred/v2/errors.html -_MIN_REQUEST_INTERVAL = 0.6 # seconds between requests => ~1.6 req/s -_last_request_time = 0.0 - - -def _throttle(): - """Sleep if needed so consecutive FRED requests stay under ~2 req/s.""" - global _last_request_time - elapsed = time.monotonic() - _last_request_time - if elapsed < _MIN_REQUEST_INTERVAL: - time.sleep(_MIN_REQUEST_INTERVAL - elapsed) - _last_request_time = time.monotonic() - - -@backoff.on_exception( - backoff.expo, - requests.exceptions.RequestException, - max_time=300, - on_backoff=data_utils.print_error_info_handler, -) -def fetch_paginated_data(url, params, field_name, pagination): - """ - Fetch data from a paginated API endpoint. - - Args: - url (str): The API endpoint URL. - params (dict): Parameters to include in the API request. - field_name (str): The key in the response JSON containing the data. - pagination (Union[bool, int]): Control pagination behavior. - - If False, fetch only the first page. - - If True, fetch all available pages. - - If int, fetch up to that many pages. - - Returns: - list: A list containing all the fetched data. - """ - all_data = [] - params["offset"] = 0 # Start from the first page - pages_fetched = 0 - - while True: - _throttle() - response = requests.get(url, params=params) - response.raise_for_status() - data = response.json() - - if not data.get(field_name, []): - break - - all_data.extend(data[field_name]) - pages_fetched += 1 - - if pagination is False: - break - elif pagination > 1 and pages_fetched >= pagination: - break - - # Increment the offset for the next page - params["offset"] += params["limit"] - - return all_data - - -def fetch_all_releases(params, series_id=None, single=False): - """ - Fetch release data from the FRED API. - - Args: - params (dict): Parameters to include in the API request. - series_id (str, optional): The ID of a specific series to fetch releases for. - Required if single is True. - single (bool, optional): Control whether to fetch data for a single series. - - If False, fetch all releases. - - If True, fetch releases for the specified series_id. - - Returns: - list: A list containing all the fetched release data. - """ - url = "https://api.stlouisfed.org/fred/releases?" - pagination = True - params["limit"] = 100 - - if single: - url = "https://api.stlouisfed.org/fred/series/release?" - params["series_id"] = series_id - pagination = False - - return fetch_paginated_data(url, params, "releases", pagination) - - -def fetch_all_series(params, all_releases=None, series_id=None): - """ - Fetch series data from the FRED API. - - Args: - params (dict): Parameters to include in the API request. - all_releases (list, optional): A list of releases to fetch series data for. - If provided, fetch series data for each release in the list. - series_id (str, optional): The ID of a specific series to fetch. - Required if all_releases is None. - - Returns: - list: A list of releases with their associated series data - if all_releases is provided. - list: A list of series data if series_id is provided. - """ - url = "https://api.stlouisfed.org/fred/release/series?" - - total_series_cnt = 0 - - if all_releases: - for release in tqdm(all_releases, desc="Fetching series"): - release_id = release["id"] - params["release_id"] = release_id - params["limit"] = 100 - - logger.info(f"Fetching release_id: {release_id}") - series_fetch = fetch_paginated_data( - url, - params, - "seriess", - pagination=20, - ) - release["series"] = [ - series - for series in series_fetch - if series["popularity"] > 50 # popularity score ranges from 0 to 100 - and series["frequency_short"] - in ["D", "W", "M"] # frequency is daily, weekly, monthly - and "Not Applicable" not in series["frequency"] - and "DISCONTINUED" not in series["title"] - ] - - total_series_cnt += len(release["series"]) - - logger.info(f"Current valid series count: {total_series_cnt}") - if total_series_cnt > 500: - break - - return all_releases - else: - url = "https://api.stlouisfed.org/fred/series?" - params["series_id"] = series_id - logger.info(f"Fetching series_id: {series_id}") - series_fetch = fetch_paginated_data(url, params, "seriess", pagination=False) - - return series_fetch - - -def fetch_all_observations(params, series_id): - """ - Fetch all observations for a given series ID from the FRED. - - Parameters: - - params (dict): Include necessary API parameters like API key. - - series_id (str): The ID of the series to fetch observations for. - - Steps: - 1. Set the series ID and limit in the params. - 2. Fetch the paginated data from the API. - 3. Parse the current date and the fetch date from the last observation. - 4. Check if the latest record is at least from last month. - 5. Filter out observations with missing values and format the data. - - Returns: - - list: A list of dictionaries containing the series ID, date, and value. - """ - url = "https://api.stlouisfed.org/fred/series/observations?" - - params["series_id"] = series_id - params["limit"] = 10000 - - observations = fetch_paginated_data(url, params, "observations", pagination=True) - - current_dt = datetime.strptime(str(dates.get_date_today()), "%Y-%m-%d") - fetch_dt = datetime.strptime(observations[-1]["date"], "%Y-%m-%d") - - # a safety check: check the latest record is at least from last month - # (some monthly updated observations seems to be lagging for a few months) - one_month_ago = current_dt - relativedelta(months=1) - is_at_least_last_month = one_month_ago <= fetch_dt - - if is_at_least_last_month and len(observations) > 0: - # save only when it's not lagging and there's data in it - observations = [ - { - "id": series_id, - "date": observation["date"], - "value": float(observation["value"]), - } - for observation in observations - if observation["value"] != "." - ] - - return observations - - return None - - -def combine_dicts(dict1, dict2): - """Combine 2 dict.""" - combined_dict = {} - - # Add all keys from dict1 to combined_dict - for key, value in dict1.items(): - if key not in combined_dict: - combined_dict[key] = {} - combined_dict[key].update(value) - - # Add all keys from dict2 to combined_dict, merging nested dictionaries - for key, value in dict2.items(): - if key not in combined_dict: - combined_dict[key] = {} - combined_dict[key].update(value) - - return combined_dict - - -def fetch_all(dfq, FRED_QUESTIONS_NAMES): - """ - Fetch and process all data for given FRED questions. - - Steps: - 1. Convert FRED_QUESTIONS_NAMES to a dictionary for easy access. - 2. Fetch release, series, and observations data for each series ID. - 3. Log the total number of questions. - 4. Iterate through the fetched data, process it, and prepare a list of dictionaries for each series. - 5. Return the processed data as a pandas DataFrame. - - Returns: - - DataFrame: A DataFrame containing the processed data with detailed information for each series. - """ - current_time = dates.get_date_today() - yesterday = current_time - timedelta(days=1) - - # get the dict version of FRED_QUESTIONS_NAMES for easy acceess - fred_questions = {q["id"]: q for q in FRED_QUESTIONS_NAMES} - - # Drop nullified series from dfq so no future question sets are built on them. - # Pre-cutoff forecasts already submitted on these ids still resolve via dfr in - # the dataset resolve path, which does not depend on dfq membership. - dfq = dfq[~dfq["id"].isin(fred.NULLIFIED_IDS)] - - # get current series ids that are not in newly fetched set - dfq_dict = dfq.to_dict(orient="records") - questions_bank_dict = {q["id"]: q for q in dfq_dict} - questions_bank_dict = { - id: questions_bank_dict[id] for id in questions_bank_dict if id not in fred_questions - } - - logger.info(f"# of questions in the new list: {len(fred_questions.keys())}") - logger.info( - f"# of questions in the bank but not in the new list: {len(questions_bank_dict.keys())}" - ) - - combined_questions = combine_dicts(fred_questions, questions_bank_dict) - - logger.info(f"# of combined questions: {len(combined_questions.keys())}") - ids_to_delete = [] - # fetch release, series, and background - for series_id in combined_questions: - combined_questions[series_id]["release"] = fetch_all_releases( - PARAMS, series_id=series_id, single=True - )[0] - combined_questions[series_id]["series"] = fetch_all_series(PARAMS, series_id=series_id) - combined_questions[series_id]["observations"] = fetch_all_observations( - PARAMS, series_id=series_id - ) - if not combined_questions[series_id]["observations"]: - ids_to_delete.append(series_id) - else: - # fill in missing dates - final_resolutions_df = pd.DataFrame(combined_questions[series_id]["observations"]) - final_resolutions_df["date"] = pd.to_datetime(final_resolutions_df["date"].str[:10]) - - # Sort and remove duplicates to keep the latest entry per date - final_resolutions_df.drop_duplicates(subset=["date"], keep="last", inplace=True) - - # Reindex to fill in missing dates including weekends - all_dates = pd.date_range( - start=final_resolutions_df["date"].min(), end=yesterday, freq="D" - ) - final_resolutions_df = ( - final_resolutions_df.set_index("date") - .reindex(all_dates, method="ffill") - .reset_index() - ) - - final_resolutions_df.rename(columns={"index": "date"}, inplace=True) - final_resolutions_df["date"] = final_resolutions_df["date"].dt.strftime("%Y-%m-%d") - - final_resolutions_df = final_resolutions_df[["id", "date", "value"]] - combined_questions[series_id]["observations"] = final_resolutions_df.to_dict( - orient="records" - ) - - logger.info(f"questions-to-delete cnt because no observations fetched: {len(ids_to_delete)}") - - for id in ids_to_delete: - del combined_questions[id] - - series_list = [] - - for series_id in tqdm(combined_questions.keys(), desc="Saving fetched FRED data"): - observations = combined_questions[series_id]["observations"] - current_value = observations[-1]["value"] - - # check if this is only from the "new" FRED Questions in fred.py - # If the questions in fred.py are updated at some point in the future, - # and the questions that exist in the old fred.py but not exist in new fred.py - # will not have a question_name field (because they only have all fields defined - # in fred_questions.json - question = None - if "series_name" in combined_questions[series_id]: - series_name = combined_questions[series_id]["series_name"] - question = ( - f"Will {series_name} have increased by " - "{resolution_date} as compared to its value on {forecast_due_date}?" - ) - else: - question = combined_questions[series_id]["question"] - - release = combined_questions[series_id]["release"] - series = combined_questions[series_id]["series"][0] - series_list.append( - { - "id": series_id, - "question": question, - "background": ( - f"The notes from the release: {release.get('notes', 'N/A')}. " - f" The notes from the series: {series.get('notes', 'N/A')}. " - " Additional background of the series: " - f" 1. the units of the series: {series.get('units', 'N/A')}. " - " 2. the seasonal adjustments of the series: " - f" {series.get('seasonal_adjustment', 'N/A')} " - f" 3. the update frequency: {series.get('frequency', 'N/A')} " - ), - "market_info_resolution_criteria": "N/A", - "market_info_open_datetime": "N/A", - "market_info_close_datetime": "N/A", - "url": f"https://fred.stlouisfed.org/series/{series_id}", - "resolved": False, - "market_info_resolution_datetime": "N/A", - "fetch_datetime": dates.get_datetime_now(), - "probability": current_value, - "forecast_horizons": ( - constants.FORECAST_HORIZONS_IN_DAYS - if series["frequency_short"] != "M" - else constants.FORECAST_HORIZONS_IN_DAYS[1:] - ), - "freeze_datetime_value": current_value, - "freeze_datetime_value_explanation": ( - "The latest value released in " - f"{series['title']} from the " - f"release {release['name']}." - ), - "resolutions": observations, - } - ) - logger.info(f"Final questions count: {len(series_list)}") - - return pd.DataFrame(series_list) - - -@decorator.log_runtime -def driver(_): - """Fetch all series from FRED and then upload to gcp.""" - dfq = data_utils.get_data_from_cloud_storage(SOURCE, return_question_data=True) - - # get the FRED_QUESTIONS_NAMES provided by @zach - FRED_QUESTIONS_NAMES = fred.fred_questions - - all_questions = fetch_all(dfq, FRED_QUESTIONS_NAMES) - filenames = data_utils.generate_filenames(SOURCE) - - # Save and upload - with open(filenames["local_fetch"], "w", encoding="utf-8") as f: - # can't use `dfq.to_json` because we don't want escape chars - for record in all_questions.to_dict("records"): - json_str = json.dumps(record, ensure_ascii=False) - f.write(json_str + "\n") - - logger.info("Uploading to GCP...") - # Upload - gcp.storage.upload( - bucket_name=env.QUESTION_BANK_BUCKET, - local_filename=filenames["local_fetch"], - ) - logger.info("Done.") - - -if __name__ == "__main__": - driver(None) diff --git a/src/questions/fred/update_questions/main.py b/src/questions/fred/update_questions/main.py deleted file mode 100644 index 2e3d4ad0..00000000 --- a/src/questions/fred/update_questions/main.py +++ /dev/null @@ -1,114 +0,0 @@ -"""FRED update question script.""" - -import logging -import os -import sys - -import pandas as pd - -sys.path.append(os.path.join(os.path.dirname(__file__), "../../..")) # noqa: E402 -from helpers import constants, data_utils, decorator, env, fred # noqa: E402 - -sys.path.append(os.path.join(os.path.dirname(__file__), "../../../..")) -from utils import gcp # noqa: E402 - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -SOURCE = "fred" - - -def create_resolution_file(question, source): - """ - Create or update a resolution file for a given question. - - Args: - question (dict): The question data containing resolutions to be saved. - source (str): The source directory or prefix for the remote file storage. - """ - basename = f"{question['id']}.jsonl" - remote_filename = f"{source}/{basename}" - local_filename = "/tmp/tmp.jsonl" - - gcp.storage.download_no_error_message_on_404( - bucket_name=env.QUESTION_BANK_BUCKET, - filename=remote_filename, - local_filename=local_filename, - ) - df = pd.read_json( - local_filename, - lines=True, - dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE, - convert_dates=False, - ) - - df = pd.DataFrame(question["resolutions"]) - df = df[["id", "date", "value"]].astype(dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE) - - df.to_json(local_filename, orient="records", lines=True, date_format="iso") - gcp.storage.upload( - bucket_name=env.QUESTION_BANK_BUCKET, - local_filename=local_filename, - filename=remote_filename, - ) - - -def update_questions(dfq, dff): - """ - Update the dataframes with new or modified question data and new resolutions. - - Parameters: - - dfq (pd.DataFrame): DataFrame containing all existing questions. - - dff (pd.DataFrame): DataFrame containing all newly fetched questions. - - The function updates dfq by either replacing existing questions with new data or adding new questions. - It also appends new series to dfr for each question in all_questions_to_add. - """ - dff_list = dff.to_dict("records") - - # Drop nullified series from dfq so no future question sets are built on them. - # Pre-cutoff forecasts already submitted on these ids still resolve via dfr in - # the dataset resolve path, which does not depend on dfq membership. - dfq = dfq[~dfq["id"].isin(fred.NULLIFIED_IDS)] - - for question in dff_list: - create_resolution_file(question, SOURCE) - - del question["fetch_datetime"] - del question["probability"] - del question["resolutions"] - - # Check if the question exists in dfq - if question["id"] in dfq["id"].values: - # Case 1: Update existing question - dfq_index = dfq.index[dfq["id"] == question["id"]].tolist()[0] - for key, value in question.items(): - dfq.at[dfq_index, key] = value - else: - # Case 2: Append new question - new_q_row = pd.DataFrame([question]) - new_q_row = new_q_row.astype(constants.QUESTION_FILE_COLUMN_DTYPE) - dfq = pd.concat([dfq, new_q_row], ignore_index=True) - - return dfq - - -@decorator.log_runtime -def driver(_): - """Execute the main workflow of processing, and uploading questions.""" - # Download existing questions from cloud storage - dfq, dff = data_utils.get_data_from_cloud_storage( - SOURCE, return_question_data=True, return_fetch_data=True - ) - - # Update the existing questions - dfq = update_questions(dfq, dff) - - logger.info("Uploading to GCP...") - # Save and upload - data_utils.upload_questions(dfq, SOURCE) - logger.info("Done.") - - -if __name__ == "__main__": - driver(None) diff --git a/src/sources/_fred_questions.py b/src/sources/_fred_questions.py new file mode 100644 index 00000000..7495b6d1 --- /dev/null +++ b/src/sources/_fred_questions.py @@ -0,0 +1,1766 @@ +"""FRED predefined question list (lightweight data module). + +The canonical FRED series list lives in the lightweight metadata layer so +consumers can reach it via ``sources._metadata`` / +``SOURCE_METADATA["fred"]["questions"]`` without importing the heavy +``sources.fred`` module (requests/backoff/pandera/...). +""" + +# flake8: noqa: B950 + +FRED_QUESTIONS = [ + { + "id": "AAA10Y", + "series_name": "Moody's Aaa Corporate Bond Yield compared to the 10-year Treasury yield", + }, + { + "id": "ANFCI", + "series_name": "the Chicago Fed's Adjusted National Financial Conditions Index", + }, + { + "id": "BAA10Y", + "series_name": "Moody's Seasoned Baa Corporate Bond Yield compared to the 10-year Treasury yield", + }, + { + "id": "BAMLC0A0CM", + "series_name": "the option-adjusted spread of the ICE BofA Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A0CMEY", + "series_name": "the effective yield of the ICE BofA Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A1CAAA", + "series_name": "the option-adjusted spread of securities with an investment grade rating of AAA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A1CAAAEY", + "series_name": "the effective yield of securities with an investment grade rating of AAA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A2CAA", + "series_name": "the option-adjusted spread of securities with an investment grade rating of AA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A2CAAEY", + "series_name": "the effective yield of securities with an investment grade rating of AA in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A3CA", + "series_name": "the option-adjusted spread of securities with an investment grade rating of A in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A3CAEY", + "series_name": "the effective yield of securities with an investment grade rating of A in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A4CBBB", + "series_name": "the option-adjusted spread of securities with an investment grade rating of BBB in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC0A4CBBBEY", + "series_name": "the effective yield of securities with an investment grade rating of BBB in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLC4A0C710YEY", + "series_name": "the effective yield of securities with a remaining term to maturity of 7-10 years in the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLCC0A0CMTRIV", + "series_name": "the total return of the ICE BofA US Corporate Index, which tracks the performance of corporate debt issued in the US domestic market,", + }, + { + "id": "BAMLEMCBPIOAS", + "series_name": "the option-adjusted spread for the ICE BofA Emerging Markets Corporate Plus Index, which tracks the performance of emerging markets non-sovereign debt within major domestic and Eurobond markets,", + }, + { + "id": "BAMLEMHBHYCRPIOAS", + "series_name": "the option-adjusted spread for the ICE BofA High Yield Emerging Markets Corporate Plus Index, which tracks the performance of emerging markets securities rated BB1 or lower within major domestic and Eurobond markets,", + }, + { + "id": "BAMLH0A0HYM2", + "series_name": "the option-adjusted spread for the ICE BofA US High Yield Index, which tracks the performance of corporate debt denominated below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A0HYM2EY", + "series_name": "the effective yield of the ICE BofA US High Yield Index, which tracks the performance of corporate debt denominated below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A1HYBB", + "series_name": "the option-adjusted spread of securities with an investment grade rating of BB in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A1HYBBEY", + "series_name": "the effective yield of securities with an investment grade rating of BB in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A2HYB", + "series_name": "the option-adjusted spread of securities with an investment grade rating of B in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A2HYBEY", + "series_name": "the effective yield of securities with an investment grade rating of B in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A3HYC", + "series_name": "the option-adjusted spread of securities with an investment grade rating of CCC or below in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLH0A3HYCEY", + "series_name": "the effective yield of securities with an investment grade rating of CCC or below in the ICE BofA US High Yield Master II Index, which tracks the performance of corporate debt below investment grade in the US domestic market,", + }, + { + "id": "BAMLHE00EHYIEY", + "series_name": "the effective yield of the ICE BofA Euro High Yield Index, which tracks the performance of below investment grade corporate debt issued in the euro domestic or eurobond markets,", + }, + { + "id": "BAMLHE00EHYIOAS", + "series_name": "the option-adjusted spread of the ICE BofA Euro High Yield Index, which tracks the performance of below investment grade corporate debt issued in the euro domestic or eurobond markets,", + }, + { + "id": "BAMLHYH0A0HYM2TRIV", + "series_name": "the total return of the ICE BofA US High Yield Index, which tracks the performance of below investment grade corporate debt publicly issued in the US domestic market,", + }, + { + "id": "CARACBW027SBOG", + "series_name": "the total dollar amount representing all automobile loans made by commercial banks in the US", + }, + {"id": "CASACBW027SBOG", "series_name": "the cash assets of all commercial US banks"}, + {"id": "CBBTCUSD", "series_name": "the price of Bitcoin, as measured by Coinbase,"}, + {"id": "CC4WSA", "series_name": "the 4-week moving average of insured unemployment claims"}, + { + "id": "CCLACBW027SBOG", + "series_name": "the amount of money representing all credit card loans and other revolving plans made by commercial banks in the US", + }, + {"id": "CCSA", "series_name": "the number of insured unemployment claims"}, + { + "id": "CREACBW027SBOG", + "series_name": "the amount of money representing all commercial real estate loans made by commercial banks in the US", + }, + { + "id": "D2WLTGAL", + "series_name": "the amount of money held by the US Treasury in its general account at the Federal Reserve Bank of New York", + }, + {"id": "DAAA", "series_name": "Moody's Seasoned Aaa Corporate Bond Yield"}, + {"id": "DBAA", "series_name": "Moody's Seasoned Baa Corporate Bond Yield"}, + {"id": "DCOILBRENTEU", "series_name": "the price of Brent crude oil"}, + { + "id": "DCOILWTICO", + "series_name": "the price of West Texas Intermediate (WTI - Cushing) crude oil", + }, + {"id": "DEXCAUS", "series_name": "the spot exchange rate of Canadian dollars to US dollars"}, + { + "id": "DEXCHUS", + "series_name": "the spot exchange rate of Chinese yuan renminbi to US dollars", + }, + {"id": "DEXJPUS", "series_name": "the spot exchange rate of Japanese yen to US dollars"}, + {"id": "DEXKOUS", "series_name": "the spot exchange rate of South Korean won to US dollars"}, + {"id": "DEXMXUS", "series_name": "the spot exchange rate of Mexican pesos to US dollars"}, + {"id": "DEXUSEU", "series_name": "the spot exchange rate of US dollars to euros"}, + {"id": "DEXUSUK", "series_name": "the spot exchange rate of US dollars to UK pound sterling"}, + { + "id": "DFEDTARL", + "series_name": "the lower limit of the target range of the federal funds rate (interest rate) set by the Federal Open Market Committee", + }, + { + "id": "DFEDTARU", + "series_name": "the upper limit of the target range of the federal funds rate (interest rate) set by the Federal Open Market Committee", + }, + { + "id": "DFF", + "series_name": "the effective federal funds rate (interest rate)", + }, + { + "id": "DFII10", + "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis and inflation-indexed,", + }, + { + "id": "DFII20", + "series_name": "the market yield on US treasury securities at 20-year constant maturity, quoted on an investment basis and inflation-indexed,", + }, + { + "id": "DFII30", + "series_name": "the market yield on US treasury securities at 30-year constant maturity, quoted on an investment basis and inflation-indexed,", + }, + { + "id": "DFII5", + "series_name": "the market yield on US treasury securities at 5-year constant maturity, quoted on an investment basis and inflation-indexed,", + }, + { + "id": "DGS1", + "series_name": "the market yield on US treasury securities at 1-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS10", + "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS1MO", + "series_name": "the market yield on US treasury securities at 1-month constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS2", + "series_name": "the market yield on US treasury securities at 2-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS20", + "series_name": "the market yield on US treasury securities at 20-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS3", + "series_name": "the market yield on US treasury securities at 3-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS30", + "series_name": "the market yield on US treasury securities at 30-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS3MO", + "series_name": "the market yield on US treasury securities at 3-month constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS5", + "series_name": "the market yield on US treasury securities at 5-year constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS6MO", + "series_name": "the market yield on US treasury securities at 6-month constant maturity, quoted on an investment basis,", + }, + { + "id": "DGS7", + "series_name": "the market yield on US treasury securities at 7-year constant maturity, quoted on an investment basis,", + }, + {"id": "DHHNGSP", "series_name": "the spot price of Henry Hub natural gas"}, + {"id": "DJIA", "series_name": "the Dow Jones Industrial Average"}, + { + "id": "DPCREDIT", + "series_name": "the discount rate for the Federal Reserve's primary credit discount window program", + }, + { + "id": "DPRIME", + "series_name": "the Federal Reserve's Bank Prime Loan Rate, the rate posted by a majority of top US commercial banks", + }, + { + "id": "DPSACBW027SBOG", + "series_name": "the amount of money representing deposits in all US commercial banks", + }, + { + "id": "DTB1YR", + "series_name": "the Federal Reserve's 1-year secondary market treasury bill rate", + }, + { + "id": "DTB3", + "series_name": "the Federal Reserve's 3-month secondary market treasury bill rate", + }, + { + "id": "DTB4WK", + "series_name": "the Federal Reserve's 4-week secondary market treasury bill rate", + }, + { + "id": "DTB6", + "series_name": "the Federal Reserve's 6-month secondary market treasury bill rate", + }, + { + "id": "DTWEXAFEGS", + "series_name": "the Nominal Advanced Foreign Economies US Dollar Index, a weighted average of the foreign exchange value of the US dollar against a subset of broad index currencies that are advanced foreign economies,", + }, + { + "id": "DTWEXBGS", + "series_name": "the Nominal Broad US Dollar Index, a weighted average of the foreign exchange value of the US dollar against currencies of a broad group of major US trading partners,", + }, + { + "id": "ECBASSETSW", + "series_name": "the amount of money representing all central bank assets for the euro area", + }, + { + "id": "ECBDFR", + "series_name": "the European Central Bank's deposit facility rate for the euro area", + }, + { + "id": "ECBESTRVOLWGTTRMDMNRT", + "series_name": "the euro short-term rate (volume-weighted trimmed mean), a measure of the borrowing costs of banks in the euro area,", + }, + { + "id": "EFFR", + "series_name": "the effective federal funds rate (interest rate) set by the Federal Reserve", + }, + { + "id": "EXPINF10YR", + "series_name": "the Federal Reserve Bank of Cleveland's 10-year expected inflation rate", + }, + { + "id": "EXPINF1YR", + "series_name": "the Federal Reserve Bank of Cleveland's 1-year expected inflation rate", + }, + { + "id": "EXPINF2YR", + "series_name": "the Federal Reserve Bank of Cleveland's 2-year expected inflation rate", + }, + { + "id": "EXPINF30YR", + "series_name": "the Federal Reserve Bank of Cleveland's 30-year expected inflation rate", + }, + { + "id": "EXPINF5YR", + "series_name": "the Federal Reserve Bank of Cleveland's 5-year expected inflation rate", + }, + {"id": "GASDESW", "series_name": "the average price of diesel in the US"}, + {"id": "GASREGW", "series_name": "the average price of regular gas in the US"}, + { + "id": "GVZCLS", + "series_name": "the Chicago Board Options Exchange's Gold ETF Volatility Index", + }, + { + "id": "H41RESPPALDKNWW", + "series_name": "the amount of money loaned as part of the Bank Term Funding Program", + }, + { + "id": "H41RESPPALDKXAWNWW", + "series_name": "the weekly average of the amount of money loaned as part of the Bank Term Funding Program", + }, + {"id": "IC4WSA", "series_name": "the 4-week moving average of initial unemployment claims"}, + {"id": "ICSA", "series_name": "the weekly number of initial unemployment claims"}, + {"id": "IHLIDXUS", "series_name": "the number of US job postings on Indeed"}, + { + "id": "IHLIDXUSTPSOFTDEVE", + "series_name": "the number of US software development job postings on Indeed", + }, + {"id": "IORB", "series_name": "the Federal Reserve's interest rate on reserve balances"}, + { + "id": "IUDSOIA", + "series_name": "the daily Sterling Overnight Index Average, the interest rate applied to bank transactions in the British Sterling Market during off hours,", + }, + { + "id": "MMTY", + "series_name": "the yield on money market investments based on US treasury obligations", + }, + {"id": "MORTGAGE15US", "series_name": "the 15-year fixed rate mortgage average in the US"}, + {"id": "MORTGAGE30US", "series_name": "the 30-year fixed rate mortgage average in the US"}, + { + "id": "NASDAQ100", + "series_name": "the NASDAQ 100 Index, which represents the daily index value at market close,", + }, + { + "id": "NASDAQCOM", + "series_name": "the NASDAQ Composite Index, which represents the daily index value at market close,", + }, + { + "id": "NDR12MCD", + "series_name": "the US national deposit rate for 12-month certificates of deposit (CDs)", + }, + {"id": "NFCI", "series_name": "the Chicago Fed's National Financial Conditions Index"}, + { + "id": "NFCICREDIT", + "series_name": "the Chicago Fed's National Financial Conditions Credit Subindex", + }, + { + "id": "NFCILEVERAGE", + "series_name": "the Chicago Fed's National Financial Conditions Leverage Subindex", + }, + { + "id": "NFCIRISK", + "series_name": "the Chicago Fed's National Financial Conditions Risk Subindex", + }, + { + "id": "NIKKEI225", + "series_name": "the Nikkei 225 Stock Average, which represent the daily index value at market close,", + }, + {"id": "OBFR", "series_name": "the Federal Reserve's overnight bank funding rate"}, + {"id": "OBMMIFHA30YF", "series_name": "the 30-year fixed rate FHA mortgage index"}, + {"id": "OBMMIJUMBO30YF", "series_name": "the 30-year fixed rate jumbo mortgage index"}, + {"id": "OBMMIVA30YF", "series_name": "the 30-year fixed rate Veterans Affairs mortgage index"}, + { + "id": "OVXCLS", + "series_name": "the Chicago Board Options Exchange's Crude Oil ETF Volatility Index", + }, + { + "id": "REAINTRATREARAT10Y", + "series_name": "the Federal Reserve Bank of Cleveland's estimate for the 10-year real interest rate", + }, + { + "id": "REAINTRATREARAT1YE", + "series_name": "the Federal Reserve Bank of Cleveland's estimate for the 1-year real interest rate", + }, + { + "id": "RESPPANWW", + "series_name": "the total dollar amount of assets held by all US Federal Reserve banks", + }, + { + "id": "RESPPLLOPNWW", + "series_name": "the total weekly remittance of earnings by the Federal Reserve to the US Treasury", + }, + { + "id": "RIFSPPFAAD90NB", + "series_name": "the 90-day AA Financial Commercial Paper Interest Rate", + }, + { + "id": "RPONTSYD", + "series_name": "the aggregated daily value of US Treasury securities repurchased overnight by the Federal Reserve in temporary open market operations ", + }, + { + "id": "RRPONTSYAWARD", + "series_name": "the award rate of US Treasury securities sold by the Federal Reserve in overnight temporary open market operations", + }, + { + "id": "RRPONTSYD", + "series_name": "the aggregated daily value of US Treasury securities sold by the Federal Reserve in temporary open market operations", + }, + { + "id": "RRPONTTLD", + "series_name": "the aggregated daily value of securities sold by the Federal Reserve in temporary open market operations", + }, + { + "id": "SNDR", + "series_name": "the aggregated value of US national average interest rates for savings accounts", + }, + {"id": "SOFR", "series_name": "the Federal Reserve's Secured Overnight Financing Rate"}, + { + "id": "SOFR180DAYAVG", + "series_name": "the 180-day average of the Federal Reserve's Secured Overnight Financing Rate", + }, + { + "id": "SOFR30DAYAVG", + "series_name": "the 30-day average of the Federal Reserve's Secured Overnight Financing Rate", + }, + { + "id": "SOFR90DAYAVG", + "series_name": "the 90-day average of the Federal Reserve's Secured Overnight Financing Rate", + }, + { + "id": "SOFRINDEX", + "series_name": "the Federal Reserve's SOFR (Secured Overnight Financing Rate) Index", + }, + { + "id": "SP500", + "series_name": "the S&P 500, which represents the daily index value at market close", + }, + {"id": "STLFSI4", "series_name": "the St. Louis Fed Financial Stress Index"}, + { + "id": "SWPT", + "series_name": "the weekly value of central bank liquidity swaps held by the Federal Reserve", + }, + { + "id": "T10Y2Y", + "series_name": "the yield spread between 10-year and 2-year US Treasury bonds", + }, + { + "id": "T10Y3M", + "series_name": "the yield spread between 10-year and 3-month US Treasury bonds", + }, + { + "id": "T10YFF", + "series_name": "the yield spread between the 10-year US Treasury bond and the Effective Federal Funds Rate (interest rate)", + }, + {"id": "T10YIE", "series_name": "the US' 10-year breakeven inflation rate"}, + {"id": "T5YIE", "series_name": "the US' 5-year breakeven inflation rate"}, + {"id": "T5YIFR", "series_name": "the US' 5-year forward inflation expectation rate"}, + {"id": "THREEFYTP10", "series_name": "the term premium on a 10-year zero-coupon bond"}, + { + "id": "TLAACBW027SBOG", + "series_name": "the total dollar amount of assets held by all US commercial banks", + }, + { + "id": "TMBACBW027SBOG", + "series_name": "the total dollar amount of mortgage-backed securities held by all US commercial banks", + }, + { + "id": "TOTBKCR", + "series_name": "the total dollar amount of bank credit held by all US commercial banks", + }, + { + "id": "TOTCI", + "series_name": "the total dollar amount representing all commercial and industrial loans made by commercial banks in the US", + }, + { + "id": "TOTLL", + "series_name": "the total dollar amount representing all loans and leases in bank credit made by commercial banks in the US", + }, + { + "id": "TREAST", + "series_name": "the total value of US Treasury securities held by the Federal Reserve", + }, + { + "id": "USEPUINDXD", + "series_name": "the Economic Policy Uncertainty Index for the US", + }, + {"id": "VIXCLS", "series_name": "the Chicago Board Options Exchange's Volatility Index"}, + { + "id": "VXVCLS", + "series_name": "the Chicago Board Options Exchange's S&P 500 3-Month Volatility Index", + }, + { + "id": "WALCL", + "series_name": "the total dollar amount of assets held by all US Federal Reserve banks", + }, + { + "id": "WDTGAL", + "series_name": "the total dollar amount of deposits in the US Treasury's general accounts of Federal Reserve Banks, other than reserve balances,", + }, + {"id": "WEI", "series_name": "the Weekly Economic Index (Lewis-Mertens-Stock)"}, + { + "id": "WGS10YR", + "series_name": "the market yield on US treasury securities at 10-year constant maturity, quoted on an investment basis,", + }, + { + "id": "WGS1YR", + "series_name": "the market yield on US treasury securities at 1-year constant maturity, quoted on an investment basis,", + }, + { + "id": "WLCFLL", + "series_name": "the weekly dollar amount of loans made by the Federal Reserve under its liquidity and credit facilities", + }, + { + "id": "WLCFLPCL", + "series_name": "the weekly dollar amount of loans made under the primary credit lending program by the Federal Reserve", + }, + { + "id": "WLODLL", + "series_name": "the weekly dollar amount of balances in the accounts of depository institutions in the Federal Reserve Banks", + }, + { + "id": "WLRRAL", + "series_name": "the weekly dollar amount associated with Federal Reserve reverse repurchase agreements", + }, + {"id": "WM1NS", "series_name": "USD money supply as measured by M1"}, + {"id": "WM2NS", "series_name": "USD money supply as measured by M2"}, + { + "id": "WORAL", + "series_name": "the weekly dollar amount associated with Federal Reserve repurchase agreements", + }, + { + "id": "WRBWFRBL", + "series_name": "the total dollar amount of reserve balances held with Federal Reverse Banks", + }, + { + "id": "WRESBAL", + "series_name": "the weekly average of reserve balances held with Federal Reserve Banks", + }, + { + "id": "WRMFNS", + "series_name": "Retail Money Market Funds, a component of M2, a measure of USD money supply,", + }, + { + "id": "WSHOMCB", + "series_name": "the total dollar amount of mortgage-backed securities held by the US Federal Reserve Banks", + }, + { + "id": "WSHOSHO", + "series_name": "the total dollar amount of securities held by US Federal Reserve Banks", + }, + { + "id": "WTREGEN", + "series_name": "the weekly average of deposits other than reserve balances held in the US treasury's general accounts with Federal Reserve Banks", + }, + { + "id": "LNU01075379", + "series_name": "the number of US civilians employed or available for employment with no disability and 65 years old and older", + }, + { + "id": "CGRAL16O", + "series_name": "the number of US civilians employed or available for employment with a Bachelor's Degree or higher and 16 years old and older", + }, + { + "id": "ADEGL16O", + "series_name": "the number of US civilians 16 years old and older employed or available for employment with an Associate Degree", + }, + { + "id": "LNU01075600", + "series_name": "the number of US civilians 65 years old and older employed or available for employment with a disability", + }, + { + "id": "LNU01073397", + "series_name": "the number of foreign born, female US civilians employed or available for employment", + }, + { + "id": "LNU01073396", + "series_name": "the number of foreign born, male US civilians employed or available for employment", + }, + { + "id": "LNU01073415", + "series_name": "the number of native born, female US civilians employed or available for employment", + }, + { + "id": "LNU01074597", + "series_name": "the number of US civilians 16 years old and older employed or available for employment with a disability", + }, + { + "id": "CLF16OV", + "series_name": "the number of US civilians employed or available for employment", + }, + { + "id": "LNU01073395", + "series_name": "the number of foreign born US civilians employed or available for employment", + }, + { + "id": "LNS11000060", + "series_name": "the number of US civilians between 25 and 54 years of age that are employed or available for employment", + }, + { + "id": "LNU01076960", + "series_name": "the number of female US civilians employed or available for employment with a disability and between 16 and 64 years of age", + }, + { + "id": "LNU01073413", + "series_name": "the number of foreign born US civilians employed or available for employment", + }, + { + "id": "LNS11024230", + "series_name": "the number of US civilians aged 55 years and above employed or available for employment", + }, + { + "id": "LNS11000002", + "series_name": "the number of female US civilians employed or available for employment", + }, + { + "id": "TOTLL65O", + "series_name": "the number of US civilians aged 65 years and above employed or available for employment", + }, + { + "id": "LNS11000001", + "series_name": "the number of male US civilians employed or available for employment", + }, + { + "id": "LNU01076955", + "series_name": "the number of male US civilians employed or available for employment with a disability and between 16 and 64 years of age", + }, + { + "id": "LNS11000009", + "series_name": "the number of Hispanic or Latino US civilians employed or available for employment", + }, + { + "id": "LNS11000003", + "series_name": "the number of White US civilians employed or available for employment", + }, + { + "id": "LNS11000006", + "series_name": "the number of Black or African American US civilians employed or available for employment", + }, + { + "id": "TOTLL3544", + "series_name": "the number of US civilians employed or available for employment between 35 and 44 years of age", + }, + { + "id": "TOTLL5564", + "series_name": "the number of US civilians employed or available for employment between 55 and 64 years of age", + }, + { + "id": "TOTLL2534", + "series_name": "the number of US civilians employed or available for employment between 25 and 34 years of age", + }, + { + "id": "LNS11000036", + "series_name": "the number of US civilians employed or available for employment between 20 and 24 years of age", + }, + { + "id": "LNS11000012", + "series_name": "the number of US civilians employed or available for employment between 16 and 19 years of age", + }, + { + "id": "LNU01032183", + "series_name": "the number of Asian US civilians employed or available for employment", + }, + { + "id": "LNU01375600", + "series_name": "the labor force participation rate among US civilians 65 years and older with a disability", + }, + { + "id": "LNU01373414", + "series_name": "the labor force participation rate among native born, male US civilians", + }, + { + "id": "LNU01373396", + "series_name": "the labor force participation rate among foreign born, male US civilians", + }, + { + "id": "LNU01373415", + "series_name": "the labor force participation rate among native born, female US civilians", + }, + { + "id": "LNU01373397", + "series_name": "the labor force participation rate among foreign born, female US civilians", + }, + { + "id": "LNU01300003", + "series_name": "the labor force participation rate among White US civilians", + }, + { + "id": "LNU01373395", + "series_name": "the labor force participation rate among foreign born US civilians", + }, + { + "id": "LNU01300009", + "series_name": "the labor force participation rate among Hispanic or Latino US civilians", + }, + { + "id": "LNU01373413", + "series_name": "the labor force participation rate among native born US civilians", + }, + { + "id": "LNU01332183", + "series_name": "the labor force participation rate among Asian US civilians", + }, + { + "id": "CIVPART", + "series_name": "the labor force participation rate among US civilians", + }, + { + "id": "LNS11300060", + "series_name": "the labor force participation rate among US civilians between 25 and 54 years of age", + }, + { + "id": "LNU01300002", + "series_name": "the labor force participation rate among female US civilians", + }, + { + "id": "LNU01300001", + "series_name": "the labor force participation rate among male US civilians", + }, + { + "id": "LNS11324230", + "series_name": "the labor force participation rate among US civilians 55 years old and older", + }, + { + "id": "LNU01300012", + "series_name": "the labor force participation rate among US civilians between 16 and 19 years of age", + }, + { + "id": "LNU01300006", + "series_name": "the labor force participation rate among Black or African American US civilians", + }, + { + "id": "LNS11300036", + "series_name": "the labor force participation rate among US civilians between 20 and 24 years of age", + }, + { + "id": "LNU01375379", + "series_name": "the labor force participation rate among US civilians 65 years old and older with no disability", + }, + { + "id": "LNU01374597", + "series_name": "the labor force participation rate among US civilians 16 years old and older with a disability", + }, + { + "id": "LNU01327662", + "series_name": "the labor force participation rate among US civilians 25 years old and older with a Bachelor's Degree", + }, + { + "id": "LNS12600000", + "series_name": "the number of employed US civilians who usually work part time", + }, + { + "id": "LNS12500000", + "series_name": "the number of employed US civilians who usually work full time", + }, + { + "id": "LNU02075600", + "series_name": "the number of employed US civilians 65 years old and older with a disability", + }, + { + "id": "LNU02074597", + "series_name": "the number of employed US civilians 16 years old and older with a disability", + }, + { + "id": "LNU02074593", + "series_name": "the number of employed US civilians 16 years old and older with no disability", + }, + { + "id": "LNU02075379", + "series_name": "the number of employed US civilians 65 years old and older with no disability", + }, + { + "id": "CE16OV", + "series_name": "the number of employed US civilians", + }, + { + "id": "LNU02000086", + "series_name": "the number of employed US civilians between 16 and 17 years of age", + }, + { + "id": "LNS12000012", + "series_name": "the number of employed US civilians between 16 and 19 years of age", + }, + { + "id": "LNS12000088", + "series_name": "the number of employed US civilians between 18 and 19 years of age", + }, + { + "id": "LNS12000036", + "series_name": "the number of employed US civilians between 20 and 24 years of age", + }, + { + "id": "LNS12000024", + "series_name": "the number of employed US civilians 20 years old and older", + }, + { + "id": "LNS12000089", + "series_name": "the number of employed US civilians between 25 and 34 years of age", + }, + { + "id": "LNS12000060", + "series_name": "the number of employed US civilians between 25 and 54 years of age", + }, + { + "id": "LNS12000048", + "series_name": "the number of employed US civilians 25 years old and older", + }, + { + "id": "LNS12000091", + "series_name": "the number of employed US civilians between 35 and 44 years of age", + }, + { + "id": "LNS12000093", + "series_name": "the number of employed US civilians between 45 and 54 years of age", + }, + { + "id": "LNS12024230", + "series_name": "the number of employed US civilians 55 years old and older", + }, + { + "id": "LNS12034560", + "series_name": "the number of US civilians employed in agriculture and related industries", + }, + { + "id": "LNS12027714", + "series_name": "the number of self-employed, unincorporated US civilians", + }, + { + "id": "LNU02032183", + "series_name": "the number of employed Asian US civilians", + }, + { + "id": "LNS12027662", + "series_name": "the number of employed US civilians 25 years old and older with a Bachelor's Degree and higher", + }, + { + "id": "LNS12000006", + "series_name": "the number of employed Black or African American US civilians", + }, + { + "id": "LNU02032210", + "series_name": "the number of US civilians employed in construction and extraction occupations", + }, + { + "id": "LNU02032209", + "series_name": "the number of US civilians employed in farming, fishing and forestry occupations", + }, + { + "id": "LNU02073395", + "series_name": "the number of employed foreign born US civilians", + }, + { + "id": "LNS12000009", + "series_name": "the number of employed Hispanic or Latino US civilians", + }, + { + "id": "LNU02032211", + "series_name": "the number of US civilians employed in installation, maintenance and repair occupations", + }, + { + "id": "LNU02032202", + "series_name": "the number of US civilians employed in management, business, and financial operations occupations", + }, + { + "id": "LNU02032201", + "series_name": "the number of US civilians employed in management, professional, and related occupations", + }, + { + "id": "LNS12000001", + "series_name": "the number of employed male US civilians", + }, + { + "id": "LNU02073413", + "series_name": "the number of employed native born US civilians", + }, + { + "id": "LNU02032208", + "series_name": "the number of US civilians employed in natural resources, construction, and maintenance occupations", + }, + { + "id": "LNS12035019", + "series_name": "the number of US civilians employed in nonagricultural industries", + }, + { + "id": "LNU02032207", + "series_name": "the number of US civilians employed in office and administrative support occupations", + }, + { + "id": "LNS12032197", + "series_name": "the number of US civilians employed part-time for economic reasons in nonagricultural industries", + }, + { + "id": "LNS12032199", + "series_name": "the number of US civilians employed part-time for economic reasons in nonagricultural industries, who could only find part-time work", + }, + { + "id": "LNS12032200", + "series_name": "the number of employed US civilians employed part-time for noneconomic reasons in nonagricultural industries", + }, + { + "id": "LNU02032213", + "series_name": "the number of US civilians employed in production occupations", + }, + { + "id": "LNU02032212", + "series_name": "the number of US civilians employed in production, transportation and material moving occupations", + }, + { + "id": "LNU02032203", + "series_name": "the number of US civilians employed in professional and related occupations", + }, + { + "id": "LNU02032205", + "series_name": "the number of US civilians employed in sales and office occupations", + }, + { + "id": "LNU02032206", + "series_name": "the number of US civilians employed in sales and related occupations", + }, + { + "id": "LNU02032204", + "series_name": "the number of US civilians employed in service occupations", + }, + { + "id": "LNU02048984", + "series_name": "the number of incorporated self-employed US civilians", + }, + { + "id": "LNS12000003", + "series_name": "the number of employed White US civilians", + }, + { + "id": "LNS12000002", + "series_name": "the number of employed female US civilians", + }, + { + "id": "LNU02374597", + "series_name": "the employment-population ratio for US civilians 16 years and older with a disability", + }, + { + "id": "LNU02375600", + "series_name": "the employment-population ratio for US civilians 65 years and older with a disability", + }, + { + "id": "LNU02374593", + "series_name": "the employment-population ratio for US civilians 16 years and older with no disability", + }, + { + "id": "LNU02375379", + "series_name": "the employment-population ratio for US civilians 65 years and older with no disability", + }, + { + "id": "LNS12300002", + "series_name": "the employment-population ratio for female US civilians", + }, + { + "id": "LNS12327689", + "series_name": "the employment-population ratio for US civilians 25 years and older with some college or associate degree", + }, + { + "id": "LNS12327660", + "series_name": "the employment-population ratio for US civilians 25 years and older with a high school diploma", + }, + { + "id": "LNS12327659", + "series_name": "the employment-population ratio for US civilians 25 years and older with less than a high school diploma", + }, + { + "id": "EMRATIO", + "series_name": "the employment-population ratio for US civilians", + }, + { + "id": "LNS12300012", + "series_name": "the employment-population ratio for US civilians between 16 and 19 years of age", + }, + { + "id": "LNS12300060", + "series_name": "the employment-population ratio for US civilians between 25 and 54 years of age", + }, + { + "id": "LNU02332183", + "series_name": "the employment-population ratio for Asian US civilians", + }, + { + "id": "LNS12327662", + "series_name": "the employment-population ratio for US civilians 25 years and older with a Bachelor's degree and higher", + }, + { + "id": "LNS12300006", + "series_name": "the employment-population ratio for Black or African American US civilians", + }, + { + "id": "LNU02373395", + "series_name": "the employment-population ratio for foreign born US civilians", + }, + { + "id": "LNS12300009", + "series_name": "the employment-population ratio for Hispanic or Latino US civilians", + }, + { + "id": "LNS12300001", + "series_name": "the employment-population ratio for male US civilians", + }, + { + "id": "LNU02373413", + "series_name": "the employment-population ratio for native born US civilians", + }, + { + "id": "LNS12300003", + "series_name": "the employment-population ratio for White US civilians", + }, + { + "id": "LNU03074597", + "series_name": "number of unemployed US civilians 16 years and older with a disability", + }, + { + "id": "LNU03075600", + "series_name": "number of unemployed US civilians 65 years and older with a disability", + }, + { + "id": "LNU03074593", + "series_name": "number of unemployed US civilians 16 years and older with no disability", + }, + { + "id": "LNU03075379", + "series_name": "number of unemployed US civilians 65 years and older with no disability", + }, + { + "id": "UNEMPLOY", + "series_name": "number of unemployed US civilians", + }, + { + "id": "LNS13000012", + "series_name": "number of unemployed US civilians between 16 and 19 years of age", + }, + { + "id": "LNS13000036", + "series_name": "number of unemployed US civilians between 20 and 24 years of age", + }, + { + "id": "LNS13000089", + "series_name": "number of unemployed US civilians between 25 and 34 years of age", + }, + { + "id": "TOTLU2564", + "series_name": "number of unemployed US civilians between 25 and 64 years of age", + }, + { + "id": "TOTLU25O", + "series_name": "number of unemployed US civilians 25 years and older", + }, + { + "id": "TOTLU3544", + "series_name": "number of unemployed US civilians between 35 and 44 years of age", + }, + { + "id": "LNS13000093", + "series_name": "number of unemployed US civilians between 45 and 54 years of age", + }, + { + "id": "TOTLU5564", + "series_name": "number of unemployed US civilians between 55 and 64 years of age", + }, + { + "id": "TOTLU65O", + "series_name": "number of unemployed US civilians 65 years and older", + }, + { + "id": "LNU03032183", + "series_name": "number of unemployed Asian US civilians", + }, + { + "id": "ADEGU16O", + "series_name": "number of unemployed US civilians 16 years and older with an associate degree", + }, + { + "id": "ADAPU16O", + "series_name": "number of unemployed US civilians 16 years and older with an associate degree (academic program)", + }, + { + "id": "ADOPU16O", + "series_name": "number of unemployed US civilians 16 years and older with an associate degree (occupational program)", + }, + { + "id": "CGRAU16O", + "series_name": "number of unemployed US civilians 16 years and older with a Bachelor's degree or higher", + }, + { + "id": "LNS13000006", + "series_name": "number of unemployed Black or African American US civilians", + }, + { + "id": "CGBDU16O", + "series_name": "number of unemployed US civilians 16 years and older with a doctoral degree", + }, + { + "id": "CGMDU16O", + "series_name": "number of unemployed US civilians 16 years and older with a Master's degree", + }, + { + "id": "LNU03073395", + "series_name": "number of unemployed foreign born US civilians", + }, + { + "id": "HSGSU16O", + "series_name": "number of unemployed US civilians 16 years and older with a high school diploma", + }, + { + "id": "LNS13000009", + "series_name": "number of unemployed Hispanic or Latino US civilians", + }, + { + "id": "LNU03023705", + "series_name": "number of unemployed US civilians who left (as opposed to lost) their job", + }, + { + "id": "LNU03023621", + "series_name": "number of unemployed US civilians who lost (as opposed to left) their job", + }, + { + "id": "LNS13025699", + "series_name": "number of unemployed US civilians who lost their job not on layoff", + }, + { + "id": "LNS13023653", + "series_name": "number of unemployed US civilians who lost their job on layoff", + }, + { + "id": "LHSDU16O", + "series_name": "number of unemployed US civilians 16 years and older with less than a high school diploma", + }, + { + "id": "LNS13100000", + "series_name": "number of unemployed US civilians looking for full-time work", + }, + { + "id": "LNS13200000", + "series_name": "number of unemployed US civilians looking for part-time work", + }, + { + "id": "LNS13000001", + "series_name": "number of unemployed male US civilians", + }, + { + "id": "LNU03073413", + "series_name": "number of unemployed native born US civilians", + }, + { + "id": "LNU03023569", + "series_name": "number of unemployed US civilians who are new entrants", + }, + { + "id": "LNS13026638", + "series_name": "number of permanently unemployed US civilians", + }, + { + "id": "LNS13026637", + "series_name": "number of unemployed US civilians who completed temporary jobs", + }, + { + "id": "SCADU16O", + "series_name": "number of unemployed US civilians 16 years and older with some college or associate degree", + }, + { + "id": "LNS13000003", + "series_name": "number of unemployed White US civilians", + }, + { + "id": "LNS13000002", + "series_name": "number of unemployed female US civilians", + }, + { + "id": "LNU03000313", + "series_name": "number of unemployed female US civilians who maintain families", + }, + { + "id": "LNU04000006", + "series_name": "the unemployement rate for Black or African American US civilians", + }, + { + "id": "CGBD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with a Bachelor's degree", + }, + { + "id": "CGDD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with a doctoral degree", + }, + { + "id": "CGMD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with a Master's degree", + }, + { + "id": "CGPD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with a professional degree", + }, + { + "id": "LNU04032240", + "series_name": "the unemployement rate for US private wage and salary workers in education and health services", + }, + { + "id": "LNU04032233", + "series_name": "the unemployement rate for US wage and salary workers in the durable goods industry", + }, + { + "id": "LNU04032231", + "series_name": "the unemployement rate for US private wage and salary workers in the construction industry", + }, + { + "id": "LNU04032224", + "series_name": "the unemployement rate for US civilians in construction and extraction occupations", + }, + { + "id": "LNU04032223", + "series_name": "the unemployement rate for US civilians in farming, fishing, and forestry occupations", + }, + { + "id": "LNU04032238", + "series_name": "the unemployement rate for US private wage and salary workers in the financial activities industry", + }, + { + "id": "LNU04073395", + "series_name": "the unemployement rate for foreign born US civilians", + }, + { + "id": "LNS14100000", + "series_name": "the unemployement rate for US full-time workers", + }, + { + "id": "HSGS16O", + "series_name": "the unemployement rate for US civilians 16 years and older with a high school diploma", + }, + { + "id": "LNU04000009", + "series_name": "the unemployement rate for Hispanic or Latino US civilians", + }, + { + "id": "LNU04032237", + "series_name": "the unemployement rate for US private wage and salary workers in the information industry", + }, + { + "id": "LNU04032225", + "series_name": "the unemployement rate for US civilians in installation, maintenance, and repair occupations", + }, + { + "id": "LNS14023705", + "series_name": "the unemployement rate for US civilians who left (as opposed to lost) their job", + }, + { + "id": "LNU04032241", + "series_name": "the unemployement rate for US private wage and salary workers in leisure and hospitality", + }, + { + "id": "LHSD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with less than a high school diploma", + }, + { + "id": "LNU04032232", + "series_name": "the unemployement rate for US private wage and salary workers in the manufacturing industry", + }, + { + "id": "LNU04032215", + "series_name": "the unemployement rate for US civilians in management, professional, and related occupations", + }, + { + "id": "LNU04032216", + "series_name": "the unemployement rate for US civilians in management, business, and financial operations occupations", + }, + { + "id": "LNS14000001", + "series_name": "the unemployement rate for male US civilians", + }, + { + "id": "LNU04073413", + "series_name": "the unemployement rate for native born US civilians", + }, + { + "id": "LNS14023569", + "series_name": "the unemployement rate for US civilians who are new entrants", + }, + { + "id": "LNU04032229", + "series_name": "the unemployement rate for US private wage and salary workers in nonagriculture occupations", + }, + { + "id": "LNU04032234", + "series_name": "the unemployement rate for US private wage and salary workers in the non durable goods industry", + }, + { + "id": "LNU04032221", + "series_name": "the unemployement rate for US civilians in office and administrative support occupations", + }, + { + "id": "LNS14200000", + "series_name": "the unemployement rate for US part-time workers", + }, + { + "id": "LNU04032227", + "series_name": "the unemployement rate for US civilians in production occupations", + }, + { + "id": "LNU04032239", + "series_name": "the unemployement rate for US private wage and salary workers in the professional and business services industry", + }, + { + "id": "LNU04032226", + "series_name": "the unemployement rate for US civilians in production, transportation and material moving occupations", + }, + { + "id": "LNU04032217", + "series_name": "the unemployement rate for US civilians in professional and related occupations", + }, + { + "id": "LNS14023557", + "series_name": "the unemployement rate for US reentrants to labor force", + }, + { + "id": "LNU04032219", + "series_name": "the unemployement rate for US civilians in the sales and office occupations", + }, + { + "id": "LNU04032218", + "series_name": "the unemployement rate for US civilians in service occupations", + }, + { + "id": "SCAD16O", + "series_name": "the unemployement rate for US civilians 16 years and older with some college or associate degree", + }, + { + "id": "LNU04032228", + "series_name": "the unemployement rate for US civilians in transportation and material moving occupations", + }, + { + "id": "LNU04032236", + "series_name": "the unemployement rate for US wage and salary workers in transportation and utilities industries", + }, + { + "id": "LNU04000003", + "series_name": "the unemployement rate for White US civilians", + }, + { + "id": "LNU04075600", + "series_name": "the unemployement rate for US civilians 65 years and older with a disability", + }, + { + "id": "LNU04074597", + "series_name": "the unemployement rate for US civilians 16 years and older with a disability", + }, + { + "id": "LNU04074593", + "series_name": "the unemployement rate for US civilians 16 years and older with no disability", + }, + { + "id": "LNU04075379", + "series_name": "the unemployement rate for US civilians 65 years and older with no disability", + }, + { + "id": "LNS14000002", + "series_name": "the unemployement rate for female US civilians", + }, + { + "id": "LNU04000313", + "series_name": "the unemployement rate for female US civilians who maintain families", + }, + { + "id": "UNRATE", + "series_name": "the unemployement rate for US civilian labor force", + }, + { + "id": "LNU04000012", + "series_name": "the unemployement rate for US civilians between 16 and 19 years of age", + }, + { + "id": "LNS14000036", + "series_name": "the unemployement rate for US civilians between 20 and 24 years of age", + }, + { + "id": "LNS14000089", + "series_name": "the unemployement rate for US civilians between 25 and 34 years of age", + }, + { + "id": "LNS14000060", + "series_name": "the unemployement rate for US civilians between 25 and 54 years of age", + }, + { + "id": "TOTL2564", + "series_name": "the unemployement rate for US civilians between 25 and 64 years of age", + }, + { + "id": "LNS14000091", + "series_name": "the unemployement rate for US civilians between 35 and 44 years of age", + }, + { + "id": "LNS14000093", + "series_name": "the unemployement rate for US civilians between 45 sand 54 years of age", + }, + { + "id": "LNU04000095", + "series_name": "the unemployement rate for US civilians between 55 and 64 years of age", + }, + { + "id": "LNU04000097", + "series_name": "the unemployement rate for US civilians 65 years and older", + }, + { + "id": "LNS14032183", + "series_name": "the unemployement rate for Asian US civilians", + }, + { + "id": "ADEG16O", + "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree", + }, + { + "id": "ADAP16O", + "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree (academic program)", + }, + { + "id": "ADOP16O", + "series_name": "the unemployement rate for US civilians 16 years and older with an associate degree (occupational program)", + }, + { + "id": "LNU05000003", + "series_name": "the number of White US civilians not in the labor force", + }, + { + "id": "LNU05074597", + "series_name": "the number of US civilians 16 years and older with a disability who are not in the labor force", + }, + { + "id": "LNU05075600", + "series_name": "the number of US civilians 65 years and older with a disability who are not in the labor force", + }, + { + "id": "LNU05074593", + "series_name": "the number of US civilians 16 years and older with no disability who are not in the labor force", + }, + { + "id": "LNU05075379", + "series_name": "the number of US civilians 65 years and older with no disability who are not in the labor force", + }, + { + "id": "LNU05000002", + "series_name": "the number of female US civilians not in the labor force", + }, + { + "id": "LNU05000001", + "series_name": "the number of male US civilians not in the labor force", + }, + { + "id": "LNU05026640", + "series_name": "the number of male US civilians not in the labor force who want a job now", + }, + { + "id": "LNU05026641", + "series_name": "the number of female US civilians not in the labor force who want a job now", + }, + { + "id": "LNU05000009", + "series_name": "the number of Hispanic or Latino US civilians not in the labor force", + }, + { + "id": "LNU05000006", + "series_name": "the number of Black or African American US civilians not in the labor force", + }, + { + "id": "LNU05000012", + "series_name": "the number of US civilians between 16 and 19 not in the labor force", + }, + { + "id": "LNU05000000", + "series_name": "the number of US civilians not in the labor force", + }, + { + "id": "LNU05073395", + "series_name": "the number of foreign born US civilians not in the labor force", + }, + { + "id": "LNU05032183", + "series_name": "the number of Asian US civilians not in the labor force", + }, + { + "id": "LNU05073413", + "series_name": "the number of native born US civilians not in the labor force", + }, + { + "id": "LNS11300012", + "series_name": "the labor force participation rate of US civilians between 16 and 19 years of age", + }, + { + "id": "LNS11300006", + "series_name": "the labor force participation rate of Black or African American US civilians", + }, + { + "id": "LNS11327662", + "series_name": "the labor force participation rate of US civilians 25 years and older with a Bachelor's degree and higher", + }, + { + "id": "LNS11300003", + "series_name": "the labor force participation rate of White US civilians", + }, + { + "id": "LNS11327660", + "series_name": "the labor force participation rate of US civilians 25 years and older with a high school diploma", + }, + { + "id": "LNS11300009", + "series_name": "the labor force participation rate of Hispanic or Latino US civilians", + }, + { + "id": "LNS11327689", + "series_name": "the labor force participation rate of US civilians 25 years and older with some college or associate degree", + }, + { + "id": "LNS11300002", + "series_name": "the labor force participation rate of female US civilians", + }, + { + "id": "LNS11300001", + "series_name": "the labor force participation rate of male US civilians", + }, + { + "id": "LNS12026620", + "series_name": "the percentage of employed US civilians who have more than one job", + }, + { + "id": "LNU02026631", + "series_name": "the number of US civilians who have more than one full-time job", + }, + { + "id": "LNU02026625", + "series_name": "the number of US civilians who have one full-time and at least one part-time job", + }, + { + "id": "LNS12026619", + "series_name": "the number of US civilians who have more than one job", + }, + { + "id": "LNU02026628", + "series_name": "the numer of US civilians who have at least two part-time jobs", + }, + { + "id": "LNU02026623", + "series_name": "the number of female US civilians who have more than one job", + }, + { + "id": "LNU02026624", + "series_name": "the percentage of employed, female US civilians who have more than one job", + }, + { + "id": "LNU02026622", + "series_name": "the percentage of employed, male US civilians who have more than one job", + }, + { + "id": "LNU02026621", + "series_name": "the number of male US civilians who have more than one job", + }, + { + "id": "UEMPMEAN", + "series_name": "the average number of weeks that US civilians have been unemployed", + }, + { + "id": "LNU03008276", + "series_name": "the median number of weeks that US civilians have been unemployed", + }, + { + "id": "LNU03008636", + "series_name": "the number of US civilians who have been unemployed for 27 weeks or more", + }, + { + "id": "LNU03008396", + "series_name": "the number of US civilians who have been unemployed for 5 weeks or less", + }, + { + "id": "LNS13025703", + "series_name": "the percentage of unemployed US civilians who have been unemployed for 27 weeks or more", + }, + { + "id": "LNU03008756", + "series_name": "the nubmer of US civilians who have been unemployed for 5 to 14 weeks", + }, + { + "id": "LNS13008397", + "series_name": "the percentage of unemployed US civilians who have been unemployed for less than 5 weeks", + }, + { + "id": "LNS13023622", + "series_name": "the percentage of unemployed US civilians who have lost (as opposed to left) their job", + }, + { + "id": "LNS13023706", + "series_name": "the percentage of unemployed US civilians who have left (as opposed to lost) their job", + }, + { + "id": "LNS13023654", + "series_name": "the percentage of unemployed US civilians who have lost their job on layoff", + }, + { + "id": "LNS13026511", + "series_name": "the percentage of unemployed US civilians who have not lost their job on layoff", + }, + { + "id": "LNS13023558", + "series_name": "the percentage of unemployed US civilians who are reentrants", + }, + { + "id": "LNS13023570", + "series_name": "the percentage of unemployed US civilians who are new entrants", + }, + { + "id": "LNS17800000", + "series_name": "the number of US civilians who went from 'employed' to 'not in labor force'", + }, + { + "id": "LNS17900000", + "series_name": "the number of US civilians who went from 'unemployed' to 'not in labor force'", + }, + { + "id": "LNS17000000", + "series_name": "the number of US civilians who remain employed", + }, + { + "id": "LNS17400000", + "series_name": "the number of US civilians who went from 'employed' to 'unemployed'", + }, + { + "id": "LNS17100000", + "series_name": "the number of US civilians who went from 'unemployed' to 'employed'", + }, + { + "id": "LNS17200000", + "series_name": "the number of US civilians who went from 'not in labor force' to 'employed'", + }, + { + "id": "LNS17600000", + "series_name": "the number of US civilians who went from 'not in labor force' to 'unemployed'", + }, + { + "id": "LNS17500000", + "series_name": "the number of US civilians who remained unemployed", + }, + { + "id": "LNS18000000", + "series_name": "the number of US civilians who remained 'not in labor force'", + }, + { + "id": "AWHAETP", + "series_name": "the average weekly hours of US employees in the private sector", + }, + { + "id": "CES0600000010", + "series_name": "the total number of female US employees in goods-producing businesses", + }, + { + "id": "CES1021100001", + "series_name": "the total number of US employees in oil and gas extraction businesses", + }, + { + "id": "USMINE", + "series_name": "the total number of US employees in mining and logging businesses", + }, + { + "id": "AWHAEMAL", + "series_name": "the average weekly hours of US employees in mining and logging businesses", + }, + { + "id": "CEU1000000011", + "series_name": "the average weekly earnings of US employees in mining and logging businesses", + }, + { + "id": "CEU1000000010", + "series_name": "the total number of female US employees in mining and logging businesses", + }, + { + "id": "AWHAEGP", + "series_name": "the average weekly hours of US employees in goods-producing businesses", + }, + { + "id": "AWHAECON", + "series_name": "the average weekly hours of US employees in construction businesses", + }, + { + "id": "CES2000000039", + "series_name": "the female US employees-to-all US employees ratio in construction businesses", + }, + { + "id": "MANEMP", + "series_name": "the number of US employees in manufacturing", + }, + { + "id": "CES3000000010", + "series_name": "the number of female US employees in manufacturing", + }, + { + "id": "AWHAEDG", + "series_name": "the average weekly hours of US employees in durable goods businesses", + }, + { + "id": "DMANEMP", + "series_name": "the total number of US employees in durable goods businesses", + }, + { + "id": "CES3133400001", + "series_name": "the total number of US employees in computer and electronic product manufacturing", + }, + { + "id": "CES3133440001", + "series_name": "the total number of US employees in semiconductor and other electronic component manufacturing", + }, + { + "id": "CES3133300001", + "series_name": "the total number of US employees in machinery manufacturing", + }, + { + "id": "CES3132100001", + "series_name": "the total number of US employees in wood product manufacturing", + }, + { + "id": "CES3133100001", + "series_name": "the total number of US employees in primary metal manufacturing", + }, + { + "id": "CES3133660001", + "series_name": "the total number of US employees in ship and boat building businesses", + }, + { + "id": "CEU3100000004", + "series_name": "the average weekly overtime hours of US employees in durable goods businesses", + }, + { + "id": "CES3133420001", + "series_name": "the total number of US employees in communications and equipment manufacturing", + }, + { + "id": "CES3100000010", + "series_name": "the total number of female US employees in durable goods businesses", + }, + { + "id": "NDMANEMP", + "series_name": "the total number of US employees in nondurable goods businesses", + }, + { + "id": "CES3231100001", + "series_name": "the total number of US employees in food manufacturing", + }, + { + "id": "CES3232500001", + "series_name": "the total number of US employees in chemical manufacturing", + }, + { + "id": "CES3231500001", + "series_name": "the total number of US employees in apparel manufacturing", + }, + { + "id": "CES3231300001", + "series_name": "the total number of US employees in textile mills", + }, + { + "id": "CES3232600001", + "series_name": "the total number of US employees in plastics and rubber products manufacturing", + }, + { + "id": "CES3232400001", + "series_name": "the total number of US employees in petroleum and coal products manufacturing", + }, + { + "id": "AWHAENDG", + "series_name": "the average weekly hours of US employees in nondurable goods businesses", + }, +] diff --git a/src/sources/_metadata.py b/src/sources/_metadata.py index 9341f479..d80e572e 100644 --- a/src/sources/_metadata.py +++ b/src/sources/_metadata.py @@ -9,6 +9,8 @@ from _fb_types import NullifiedQuestion, SourceType from helpers.constants import BENCHMARK_START_DATE_DATETIME_DATE +from ._fred_questions import FRED_QUESTIONS + SOURCE_METADATA = { "acled": { "source_type": SourceType.DATASET, @@ -51,6 +53,9 @@ nullification_start_date=date(2025, 11, 1), ), ], + # Predefined FRED series the fetch job pulls each run (data lives in the + # lightweight _fred_questions module so consumers needn't import sources.fred). + "questions": FRED_QUESTIONS, }, "infer": { "source_type": SourceType.MARKET, diff --git a/src/sources/fred.py b/src/sources/fred.py index 7baac88c..6bb60096 100644 --- a/src/sources/fred.py +++ b/src/sources/fred.py @@ -2,32 +2,412 @@ from __future__ import annotations -from typing import ClassVar +import logging +import time +from datetime import datetime, timedelta +from typing import Any, ClassVar -from _fb_types import NullifiedQuestion +import backoff +import pandas as pd +import pandera.pandas as pa +import requests +from dateutil.relativedelta import relativedelta +from pandera.typing import DataFrame + +from _fb_types import UpdateResult +from _schemas import FredFetchFrame, QuestionFrame +from helpers import constants, data_utils, dates from ._dataset import DatasetSource -from ._metadata import SOURCE_METADATA -NULLIFIED_QUESTIONS = { - nq.id: nq.nullification_start_date for nq in SOURCE_METADATA["fred"]["nullified_questions"] -} +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.stlouisfed.org" -NULLIFIED_IDS = [nq.id for nq in SOURCE_METADATA["fred"]["nullified_questions"]] +# FRED throttles each API key at ~2 requests/second (120/minute); exceeding it +# returns HTTP 429. Space requests out to stay safely under that limit. +# See https://fred.stlouisfed.org/docs/api/fred/v2/errors.html +_MIN_REQUEST_INTERVAL = 0.6 # seconds between requests => ~1.6 req/s class FredSource(DatasetSource): """Federal Reserve Economic Data source.""" name: ClassVar[str] = "fred" - nullified_questions: ClassVar[list[NullifiedQuestion]] = SOURCE_METADATA["fred"][ - "nullified_questions" - ] - def fetch(self, **kwargs): - """Fetch FRED data from external API.""" - raise NotImplementedError + def __init__(self) -> None: + """Initialize with request-throttling state.""" + super().__init__() + self._last_request_time = 0.0 + + # ------------------------------------------------------------------ + # Public: fetch + # ------------------------------------------------------------------ + + @pa.check_types + def fetch( + self, + *, + dfq: DataFrame[QuestionFrame] | None = None, + ) -> DataFrame[FredFetchFrame]: + """Fetch FRED series data from the API. + + Args: + dfq (DataFrame[QuestionFrame] | None): Existing question bank. + """ + self._require_api_key() + current_time = dates.get_datetime_now() + yesterday = dates.get_date_today() - timedelta(days=1) + nullified_ids = self.get_nullified_ids() + + # Build combined question dict: predefined series list + existing bank-only series. + # self.questions is auto-populated from SOURCE_METADATA["fred"]["questions"]. + fred_questions = {q["id"]: q for q in self.questions} + + questions_bank_dict: dict[str, dict] = {} + if dfq is not None and not dfq.empty: + # Drop nullified series so no future question sets are built on them. + # Pre-cutoff forecasts already submitted on these ids still resolve via dfr + # in the dataset resolve path, which does not depend on dfq membership. + dfq_filtered = dfq[~dfq["id"].isin(nullified_ids)] + dfq_dict = {q["id"]: q for q in dfq_filtered.to_dict(orient="records")} + questions_bank_dict = {id: dfq_dict[id] for id in dfq_dict if id not in fred_questions} + + logger.info(f"# of questions in the new list: {len(fred_questions)}") + logger.info( + f"# of questions in the bank but not in the new list: {len(questions_bank_dict)}" + ) + + combined_questions = self._combine_dicts(fred_questions, questions_bank_dict) + logger.info(f"# of combined questions: {len(combined_questions)}") + + # Fetch API data for each series. + ids_to_delete = [] + for series_id in combined_questions: + combined_questions[series_id]["release"] = self._fetch_release(series_id) + combined_questions[series_id]["series"] = self._fetch_series_info(series_id) + combined_questions[series_id]["observations"] = self._fetch_observations(series_id) + + if not combined_questions[series_id]["observations"]: + ids_to_delete.append(series_id) + else: + combined_questions[series_id]["observations"] = self._forward_fill_observations( + combined_questions[series_id]["observations"], yesterday + ) + + logger.info( + f"questions-to-delete cnt because no observations fetched: {len(ids_to_delete)}" + ) + for id in ids_to_delete: + del combined_questions[id] + + # Transform to FredFetchFrame rows. + rows = [ + self._transform_series( + series_id=series_id, + combined_question=combined_questions[series_id], + current_time=current_time, + ) + for series_id in combined_questions + ] + + logger.info(f"Final questions count: {len(rows)}") + return pd.DataFrame(rows) + + # ------------------------------------------------------------------ + # Public: update + # ------------------------------------------------------------------ + + @pa.check_types + def update( + self, + dfq: DataFrame[QuestionFrame], + dff: DataFrame[FredFetchFrame], + **kwargs: Any, + ) -> UpdateResult: + """Process fetched data into updated questions and resolution files. + + Args: + dfq (DataFrame[QuestionFrame]): Existing questions. + dff (DataFrame[FredFetchFrame]): Freshly fetched data. + """ + # Drop nullified series so no future question sets are built on them. + # Pre-cutoff forecasts already submitted on these ids still resolve via dfr + # in the dataset resolve path, which does not depend on dfq membership. + nullified_ids = self.get_nullified_ids() + dfq = dfq[~dfq["id"].isin(nullified_ids)].copy() + resolution_files: dict[str, pd.DataFrame] = {} + + for question in dff.to_dict("records"): + question_id = str(question["id"]) + + # Build resolution file from embedded observations. + df_res = pd.DataFrame(question["resolutions"]) + df_res = df_res[["id", "date", "value"]] + resolution_files[question_id] = df_res + + # Strip transient fields (not part of QuestionFrame). + del question["fetch_datetime"] + del question["probability"] + del question["resolutions"] + + # Upsert into dfq. + if question["id"] in dfq["id"].values: + dfq_index = dfq.index[dfq["id"] == question["id"]].tolist()[0] + for key, value in question.items(): + dfq.at[dfq_index, key] = value + else: + dfq = pd.concat([dfq, pd.DataFrame([question])], ignore_index=True) + + return UpdateResult( + dfq=dfq, + resolution_files=resolution_files, + ) + + # ------------------------------------------------------------------ + # Private: request throttling + # ------------------------------------------------------------------ + + def _throttle(self) -> None: + """Sleep if needed so consecutive FRED requests stay under ~2 req/s.""" + elapsed = time.monotonic() - self._last_request_time + if elapsed < _MIN_REQUEST_INTERVAL: + time.sleep(_MIN_REQUEST_INTERVAL - elapsed) + self._last_request_time = time.monotonic() + + # ------------------------------------------------------------------ + # Private: API calls + # ------------------------------------------------------------------ + + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=300, + on_backoff=data_utils.print_error_info_handler, + ) + def _fetch_paginated_data( + self, + url: str, + params: dict, + field_name: str, + pagination: bool | int, + ) -> list: + """Fetch data from a paginated FRED API endpoint. + + Args: + url (str): The API endpoint URL. + params (dict): Parameters for the request. + field_name (str): Key in response JSON containing the data. + pagination (bool | int): False=1 page, True=all, int=up to N pages. + """ + all_data = [] + params["offset"] = 0 + pages_fetched = 0 + + while True: + self._throttle() + response = requests.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get(field_name, []): + break + + all_data.extend(data[field_name]) + pages_fetched += 1 + + if pagination is False: + break + elif isinstance(pagination, int) and pagination > 1 and pages_fetched >= pagination: + break + + params["offset"] += params["limit"] + + return all_data + + def _fetch_release(self, series_id: str) -> dict: + """Fetch release info for a single series. + + Args: + series_id (str): FRED series ID. + """ + api_key = self._require_api_key() + params = { + "api_key": api_key, + "file_type": "json", + "series_id": series_id, + } + results = self._fetch_paginated_data( + url=f"{_BASE_URL}/fred/series/release?", + params=params, + field_name="releases", + pagination=False, + ) + return results[0] + + def _fetch_series_info(self, series_id: str) -> list[dict]: + """Fetch series metadata for a single series. + + Args: + series_id (str): FRED series ID. + """ + api_key = self._require_api_key() + params = { + "api_key": api_key, + "file_type": "json", + "series_id": series_id, + } + return self._fetch_paginated_data( + url=f"{_BASE_URL}/fred/series?", + params=params, + field_name="seriess", + pagination=False, + ) + + def _fetch_observations(self, series_id: str) -> list[dict] | None: + """Fetch all observations for a series. + + Returns the cleaned observations, or None when the series' most recent + observation is more than a month stale (some monthly series lag). + + Args: + series_id (str): FRED series ID. + """ + api_key = self._require_api_key() + params = { + "api_key": api_key, + "file_type": "json", + "series_id": series_id, + "limit": 10000, + } + observations = self._fetch_paginated_data( + url=f"{_BASE_URL}/fred/series/observations?", + params=params, + field_name="observations", + pagination=True, + ) + + current_dt = datetime.strptime(str(dates.get_date_today()), "%Y-%m-%d") + fetch_dt = datetime.strptime(observations[-1]["date"], "%Y-%m-%d") + + # Safety check: the latest record must be at least from last month. + one_month_ago = current_dt - relativedelta(months=1) + is_at_least_last_month = one_month_ago <= fetch_dt + + if is_at_least_last_month and len(observations) > 0: + # Filter out missing values and convert to float. + return [ + { + "id": series_id, + "date": observation["date"], + "value": float(observation["value"]), + } + for observation in observations + if observation["value"] != "." + ] + + return None + + # ------------------------------------------------------------------ + # Private: data transformation + # ------------------------------------------------------------------ + + @staticmethod + def _combine_dicts(dict1: dict, dict2: dict) -> dict: + """Combine two dicts of dicts, merging nested values. + + Args: + dict1 (dict): Primary dict (e.g. new hardcoded fred_questions). + dict2 (dict): Secondary dict (e.g. bank-only questions). + """ + combined: dict = {} + for key, value in dict1.items(): + if key not in combined: + combined[key] = {} + combined[key].update(value) + for key, value in dict2.items(): + if key not in combined: + combined[key] = {} + combined[key].update(value) + return combined + + @staticmethod + def _forward_fill_observations(observations: list[dict], yesterday) -> list[dict]: + """Forward-fill missing dates in observations up to yesterday. + + Args: + observations (list[dict]): Raw observations with {id, date, value}. + yesterday (date): Fill dates up to this date. + """ + df = pd.DataFrame(observations) + df["date"] = pd.to_datetime(df["date"].str[:10]) + df.drop_duplicates(subset=["date"], keep="last", inplace=True) + + all_dates = pd.date_range(start=df["date"].min(), end=yesterday, freq="D") + df = df.set_index("date").reindex(all_dates, method="ffill").reset_index() + df.rename(columns={"index": "date"}, inplace=True) + df["date"] = df["date"].dt.strftime("%Y-%m-%d") + + return df[["id", "date", "value"]].to_dict(orient="records") + + @staticmethod + def _transform_series( + series_id: str, + combined_question: dict, + current_time: str, + ) -> dict: + """Transform fetched series data into a FredFetchFrame row. + + Args: + series_id (str): FRED series ID. + combined_question (dict): Merged question data with release, series, observations. + current_time (str): ISO timestamp for fetch_datetime. + """ + observations = combined_question["observations"] + current_value = observations[-1]["value"] + release = combined_question["release"] + series = combined_question["series"][0] + + # Question text: use series_name from the hardcoded list, else existing question text. + if "series_name" in combined_question: + series_name = combined_question["series_name"] + question = ( + f"Will {series_name} have increased by " + "{resolution_date} as compared to its value on {forecast_due_date}?" + ) + else: + question = combined_question["question"] - def update(self, dfq, dff, **kwargs): - """Process fetched FRED data into questions and resolution files.""" - raise NotImplementedError + return { + "id": series_id, + "question": question, + "background": ( + f"The notes from the release: {release.get('notes', 'N/A')}. " + f" The notes from the series: {series.get('notes', 'N/A')}. " + " Additional background of the series: " + f" 1. the units of the series: {series.get('units', 'N/A')}. " + " 2. the seasonal adjustments of the series: " + f" {series.get('seasonal_adjustment', 'N/A')} " + f" 3. the update frequency: {series.get('frequency', 'N/A')} " + ), + "market_info_resolution_criteria": "N/A", + "market_info_open_datetime": "N/A", + "market_info_close_datetime": "N/A", + "url": f"https://fred.stlouisfed.org/series/{series_id}", + "resolved": False, + "market_info_resolution_datetime": "N/A", + "fetch_datetime": current_time, + "probability": current_value, + "forecast_horizons": ( + constants.FORECAST_HORIZONS_IN_DAYS + if series["frequency_short"] != "M" + else constants.FORECAST_HORIZONS_IN_DAYS[1:] + ), + "freeze_datetime_value": current_value, + "freeze_datetime_value_explanation": ( + f"The latest value released in " + f"{series['title']} from the " + f"release {release['name']}." + ), + "resolutions": observations, + } diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 0af0d240..4e7109c9 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -519,3 +519,111 @@ def make_polymarket_fetch_df(rows): if col not in df.columns: df[col] = default return df + + +# --------------------------------------------------------------------------- +# FRED-specific fixtures and factories +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def fred_source(): + """Return a FredSource instance with a fake API key.""" + src = FredSource() + src.api_key = "test-key" + return src + + +def make_fred_api_release(**overrides): + """Build a realistic FRED release API response dict.""" + base = { + "id": 18, + "realtime_start": "2026-03-18", + "realtime_end": "2026-03-18", + "name": "H.15 Selected Interest Rates", + "press_release": True, + "link": "http://www.federalreserve.gov/releases/h15/", + "notes": "For questions on the data, please contact the data source.", + } + base.update(overrides) + return base + + +def make_fred_api_series(**overrides): + """Build a realistic FRED series API response dict.""" + base = { + "id": "DGS10", + "title": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity", + "observation_start": "1962-01-02", + "observation_end": "2026-03-16", + "frequency": "Daily", + "frequency_short": "D", + "units": "Percent", + "units_short": "%", + "seasonal_adjustment": "Not Seasonally Adjusted", + "seasonal_adjustment_short": "NSA", + "last_updated": "2026-03-17 15:16:44-05", + "popularity": 98, + "notes": "H.15 Statistical Release.", + } + base.update(overrides) + return base + + +def make_fred_api_observations(series_id="DGS10", date_values=None): + """Build a list of FRED observation dicts. + + Args: + series_id (str): The series ID for the id field. + date_values (list): List of (date_str, value_str) tuples. + """ + if date_values is None: + date_values = [ + ("2026-03-14", "4.25"), + ("2026-03-15", "."), + ("2026-03-16", "4.30"), + ] + return [ + { + "realtime_start": "2026-03-18", + "realtime_end": "2026-03-18", + "date": d, + "value": v, + } + for d, v in date_values + ] + + +def make_fred_fetch_df(rows): + """Build a DataFrame matching FredFetchFrame schema. + + Each row should have at least 'id'. Missing columns get defaults. + """ + defaults = { + "question": ( + "Will X have increased by {resolution_date} as compared to " + "its value on {forecast_due_date}?" + ), + "background": "N/A", + "url": "https://fred.stlouisfed.org/series/DGS10", + "resolved": False, + "forecast_horizons": [7, 30, 90, 180, 365, 1095, 1825, 3650], + "freeze_datetime_value": 4.30, + "freeze_datetime_value_explanation": "The latest value released in X from the release Y.", + "market_info_resolution_criteria": "N/A", + "market_info_open_datetime": "N/A", + "market_info_close_datetime": "N/A", + "market_info_resolution_datetime": "N/A", + "fetch_datetime": "2026-03-18T00:00:00+00:00", + "probability": 4.30, + "resolutions": [ + {"id": "DGS10", "date": "2026-03-14", "value": 4.25}, + {"id": "DGS10", "date": "2026-03-15", "value": 4.25}, + {"id": "DGS10", "date": "2026-03-16", "value": 4.30}, + ], + } + df = pd.DataFrame(rows) + for col, default in defaults.items(): + if col not in df.columns: + df[col] = [default] * len(df) if isinstance(default, list) else default + return df diff --git a/src/tests/test_fred.py b/src/tests/test_fred.py index 3ad727bf..cb3822dc 100644 --- a/src/tests/test_fred.py +++ b/src/tests/test_fred.py @@ -1,23 +1,50 @@ -"""Tests for FRED-specific nullification rules.""" +"""Tests for FredSource: nullification rules plus fetch/update logic.""" from datetime import date +from unittest.mock import Mock, patch import pandas as pd +import pytest +from _schemas import FredFetchFrame, QuestionFrame from helpers import fred as fred_helper -from questions.fred.update_questions.main import update_questions -from sources.fred import NULLIFIED_IDS, NULLIFIED_QUESTIONS, FredSource -from tests.conftest import make_forecast_df, make_question_df, make_resolution_df +from sources.fred import FredSource + +from .conftest import ( + make_forecast_df, + make_fred_api_observations, + make_fred_api_release, + make_fred_api_series, + make_fred_fetch_df, + make_question_df, + make_resolution_df, +) + + +@pytest.fixture(autouse=True) +def _no_throttle_sleep(): + """Neutralize the FRED request throttle so tests don't really sleep.""" + with patch("sources.fred.time.sleep"): + yield + + +# --------------------------------------------------------------------------- +# Nullification definition +# --------------------------------------------------------------------------- class TestFredNullificationDefinition: """Verify retired FRED series metadata is declared correctly.""" def test_currcir_removed_from_fetch_pool(self): - assert "CURRCIR" in NULLIFIED_IDS - assert NULLIFIED_QUESTIONS["CURRCIR"] == date(2025, 11, 1) + # NULLIFIED_IDS is exposed via the backwards-compat helpers shim (from metadata), + # mirroring helpers/polymarket.py — the source module doesn't re-export it. + assert "CURRCIR" in fred_helper.NULLIFIED_IDS assert all(question["id"] != "CURRCIR" for question in fred_helper.fred_questions) + def test_ameribor_nullified(self): + assert "AMERIBOR" in fred_helper.NULLIFIED_IDS + class TestFredSourceNullification: """Test that retired FRED series nullify by forecast date.""" @@ -110,13 +137,463 @@ def test_currcir_pre_cutoff_question_with_missing_resolution_data_is_unresolved( assert bool(row["resolved"]) is False -class TestFredFetchAndUpdateRegression: - """Test that legacy CURRCIR rows are no longer carried forward.""" +# --------------------------------------------------------------------------- +# _throttle +# --------------------------------------------------------------------------- + + +class TestThrottle: + """Tests for FredSource._throttle (FRED 2 req/s rate limit).""" + + def test_sleeps_when_called_too_soon(self, fred_source): + """Sleeps the remaining interval when requests are too close together.""" + with patch("sources.fred.time.monotonic", side_effect=[100.1, 100.1]), patch( + "sources.fred.time.sleep" + ) as mock_sleep: + fred_source._last_request_time = 100.0 # elapsed = 0.1 < 0.6 + fred_source._throttle() + mock_sleep.assert_called_once() + assert abs(mock_sleep.call_args[0][0] - 0.5) < 1e-9 + + def test_no_sleep_when_interval_elapsed(self, fred_source): + """Does not sleep when enough time has already passed.""" + with patch("sources.fred.time.monotonic", side_effect=[101.0, 101.0]), patch( + "sources.fred.time.sleep" + ) as mock_sleep: + fred_source._last_request_time = 100.0 # elapsed = 1.0 > 0.6 + fred_source._throttle() + mock_sleep.assert_not_called() + + +# --------------------------------------------------------------------------- +# _combine_dicts (pure, no mocking) +# --------------------------------------------------------------------------- + + +class TestCombineDicts: + """Tests for FredSource._combine_dicts static method.""" + + def test_disjoint_dicts(self): + d1 = {"A": {"id": "A", "series_name": "Series A"}} + d2 = {"B": {"id": "B", "question": "Old question"}} + result = FredSource._combine_dicts(d1, d2) + assert set(result.keys()) == {"A", "B"} + assert result["A"]["series_name"] == "Series A" + assert result["B"]["question"] == "Old question" + + def test_overlapping_keys_merge(self): + d1 = {"A": {"id": "A", "series_name": "New name"}} + d2 = {"A": {"id": "A", "question": "Old question", "background": "Old bg"}} + result = FredSource._combine_dicts(d1, d2) + assert result["A"]["series_name"] == "New name" + assert result["A"]["question"] == "Old question" + + def test_empty_dicts(self): + assert FredSource._combine_dicts({}, {}) == {} + assert FredSource._combine_dicts({"A": {"x": 1}}, {}) == {"A": {"x": 1}} + assert FredSource._combine_dicts({}, {"B": {"y": 2}}) == {"B": {"y": 2}} + + +# --------------------------------------------------------------------------- +# _forward_fill_observations (pure, no mocking) +# --------------------------------------------------------------------------- + + +class TestForwardFillObservations: + """Tests for FredSource._forward_fill_observations static method.""" + + def test_fills_gaps(self): + observations = [ + {"id": "X", "date": "2026-03-10", "value": 1.0}, + {"id": "X", "date": "2026-03-13", "value": 2.0}, + ] + result = FredSource._forward_fill_observations(observations, date(2026, 3, 14)) + + dates_list = [r["date"] for r in result] + assert "2026-03-11" in dates_list + assert "2026-03-12" in dates_list + assert "2026-03-14" in dates_list + assert result[1]["value"] == 1.0 # 03-11 filled from 03-10 + assert result[2]["value"] == 1.0 # 03-12 filled from 03-10 + assert result[3]["value"] == 2.0 # 03-13 actual value + + def test_deduplicates_dates(self): + observations = [ + {"id": "X", "date": "2026-03-10", "value": 1.0}, + {"id": "X", "date": "2026-03-10", "value": 2.0}, + ] + result = FredSource._forward_fill_observations(observations, date(2026, 3, 10)) + assert len(result) == 1 + assert result[0]["value"] == 2.0 + + def test_extends_to_yesterday(self): + observations = [{"id": "X", "date": "2026-03-10", "value": 5.0}] + result = FredSource._forward_fill_observations(observations, date(2026, 3, 12)) + assert len(result) == 3 # 10, 11, 12 + assert result[-1]["date"] == "2026-03-12" + assert result[-1]["value"] == 5.0 + + def test_preserves_id(self): + observations = [{"id": "DGS10", "date": "2026-03-10", "value": 4.0}] + result = FredSource._forward_fill_observations(observations, date(2026, 3, 11)) + assert all(r["id"] == "DGS10" for r in result) + + +# --------------------------------------------------------------------------- +# _transform_series (pure, no mocking) +# --------------------------------------------------------------------------- + + +class TestTransformSeries: + """Tests for FredSource._transform_series static method.""" + + CURRENT_TIME = "2026-03-18T00:00:00+00:00" + + def _combined_question(self, **overrides): + base = { + "series_name": "the 10-year Treasury yield", + "release": make_fred_api_release(), + "series": [make_fred_api_series()], + "observations": [{"id": "DGS10", "date": "2026-03-16", "value": 4.30}], + } + base.update(overrides) + return base + + def test_standard_daily_series(self): + row = FredSource._transform_series("DGS10", self._combined_question(), self.CURRENT_TIME) + + assert row["id"] == "DGS10" + assert "{resolution_date}" in row["question"] + assert "{forecast_due_date}" in row["question"] + assert row["resolved"] is False + assert row["probability"] == 4.30 + assert row["freeze_datetime_value"] == 4.30 + assert row["url"] == "https://fred.stlouisfed.org/series/DGS10" + assert 7 in row["forecast_horizons"] + FredFetchFrame.validate(pd.DataFrame([row])) + + def test_monthly_series_excludes_7day(self): + series = make_fred_api_series(frequency_short="M", frequency="Monthly") + row = FredSource._transform_series( + "PAYEMS", self._combined_question(series=[series]), self.CURRENT_TIME + ) + assert 7 not in row["forecast_horizons"] + assert 30 in row["forecast_horizons"] + + def test_bank_only_series_uses_existing_question(self): + cq = self._combined_question() + del cq["series_name"] + cq["question"] = "Will old series increase?" + row = FredSource._transform_series("OLD_ID", cq, self.CURRENT_TIME) + assert row["question"] == "Will old series increase?" + + def test_background_contains_release_and_series_info(self): + row = FredSource._transform_series("DGS10", self._combined_question(), self.CURRENT_TIME) + assert "Percent" in row["background"] + assert "Not Seasonally Adjusted" in row["background"] + + def test_freeze_value_explanation(self): + row = FredSource._transform_series("DGS10", self._combined_question(), self.CURRENT_TIME) + assert "Market Yield" in row["freeze_datetime_value_explanation"] + assert "H.15" in row["freeze_datetime_value_explanation"] + + +# --------------------------------------------------------------------------- +# _fetch_observations (mock _fetch_paginated_data) +# --------------------------------------------------------------------------- + + +class TestFetchObservations: + """Tests for FredSource._fetch_observations.""" + + @patch.object(FredSource, "_fetch_paginated_data") + def test_filters_dot_values(self, mock_paginated, fred_source, freeze_today): + freeze_today(date(2026, 3, 18)) + mock_paginated.return_value = make_fred_api_observations( + date_values=[("2026-03-14", "4.25"), ("2026-03-15", "."), ("2026-03-16", "4.30")] + ) + result = fred_source._fetch_observations("DGS10") + + assert len(result) == 2 + assert all(r["value"] != "." for r in result) + assert all(isinstance(r["value"], float) for r in result) + + @patch.object(FredSource, "_fetch_paginated_data") + def test_stale_data_returns_none(self, mock_paginated, fred_source, freeze_today): + freeze_today(date(2026, 3, 18)) + mock_paginated.return_value = make_fred_api_observations( + date_values=[("2025-12-01", "100.0")] + ) + result = fred_source._fetch_observations("STALE_SERIES") + assert result is None + + @patch.object(FredSource, "_fetch_paginated_data") + def test_recent_data_returns_observations(self, mock_paginated, fred_source, freeze_today): + freeze_today(date(2026, 3, 18)) + mock_paginated.return_value = make_fred_api_observations( + date_values=[("2026-03-10", "1.5"), ("2026-03-16", "1.7")] + ) + result = fred_source._fetch_observations("DGS10") + assert len(result) == 2 + assert result[0] == {"id": "DGS10", "date": "2026-03-10", "value": 1.5} + + +# --------------------------------------------------------------------------- +# _fetch_paginated_data (mock requests.get) +# --------------------------------------------------------------------------- + + +class TestFetchPaginatedData: + """Tests for FredSource._fetch_paginated_data.""" + + def _mock_response(self, data): + resp = Mock() + resp.json.return_value = data + resp.raise_for_status = Mock() + return resp + + @patch("sources.fred.requests.get") + def test_single_page(self, mock_get, fred_source): + mock_get.return_value = self._mock_response({"releases": [{"id": 1}, {"id": 2}]}) + result = fred_source._fetch_paginated_data( + url="http://test", + params={"api_key": "k", "file_type": "json"}, + field_name="releases", + pagination=False, + ) + assert len(result) == 2 + assert mock_get.call_count == 1 + + @patch("sources.fred.requests.get") + def test_multi_page(self, mock_get, fred_source): + mock_get.side_effect = [ + self._mock_response({"data": [{"id": 1}]}), + self._mock_response({"data": [{"id": 2}]}), + self._mock_response({"data": []}), + ] + result = fred_source._fetch_paginated_data( + url="http://test", + params={"api_key": "k", "file_type": "json", "limit": 1}, + field_name="data", + pagination=True, + ) + assert len(result) == 2 + assert mock_get.call_count == 3 + + @patch("sources.fred.requests.get") + def test_max_pages(self, mock_get, fred_source): + mock_get.side_effect = [self._mock_response({"data": [{"id": i}]}) for i in range(10)] + result = fred_source._fetch_paginated_data( + url="http://test", + params={"api_key": "k", "file_type": "json", "limit": 1}, + field_name="data", + pagination=3, + ) + assert len(result) == 3 + assert mock_get.call_count == 3 + + @patch("sources.fred.requests.get") + def test_empty_first_page(self, mock_get, fred_source): + mock_get.return_value = self._mock_response({"data": []}) + result = fred_source._fetch_paginated_data( + url="http://test", + params={"api_key": "k", "file_type": "json"}, + field_name="data", + pagination=True, + ) + assert result == [] + + @patch.object(FredSource, "_throttle") + @patch("sources.fred.requests.get") + def test_throttles_before_every_request(self, mock_get, mock_throttle, fred_source): + """FRED's ~2 req/s limit is honored: _throttle runs before each request (prod fix).""" + mock_get.side_effect = [ + self._mock_response({"data": [{"id": 1}]}), + self._mock_response({"data": [{"id": 2}]}), + self._mock_response({"data": []}), + ] + fred_source._fetch_paginated_data( + url="http://test", + params={"api_key": "k", "file_type": "json", "limit": 1}, + field_name="data", + pagination=True, + ) + assert mock_throttle.call_count == mock_get.call_count + + +# --------------------------------------------------------------------------- +# fetch() (mock private API methods + patch _FRED_QUESTIONS) +# --------------------------------------------------------------------------- + + +class TestFetch: + """Tests for FredSource.fetch.""" + + @patch.object(FredSource, "questions", [{"id": "DGS10", "series_name": "10-year Treasury"}]) + @patch.object(FredSource, "_fetch_observations") + @patch.object(FredSource, "_fetch_series_info") + @patch.object(FredSource, "_fetch_release") + def test_basic_fetch(self, mock_release, mock_series, mock_obs, fred_source, freeze_today): + freeze_today(date(2026, 3, 18)) + mock_release.return_value = make_fred_api_release() + mock_series.return_value = [make_fred_api_series()] + mock_obs.return_value = [ + {"id": "DGS10", "date": "2026-03-16", "value": 4.30}, + {"id": "DGS10", "date": "2026-03-17", "value": 4.35}, + ] + + dff = fred_source.fetch(dfq=None) + + assert len(dff) == 1 + assert dff.iloc[0]["id"] == "DGS10" + # fetch threads `yesterday` (03-17) into the embedded series' forward-fill, + # and freeze/probability take the latest (yesterday's) value. + resolutions = dff.iloc[0]["resolutions"] + assert resolutions[-1]["date"] == "2026-03-17" + assert dff.iloc[0]["probability"] == 4.35 + FredFetchFrame.validate(dff) + + @patch.object( + FredSource, + "questions", + [{"id": "GOOD", "series_name": "Good series"}, {"id": "BAD", "series_name": "Bad series"}], + ) + @patch.object(FredSource, "_fetch_observations") + @patch.object(FredSource, "_fetch_series_info") + @patch.object(FredSource, "_fetch_release") + def test_series_with_no_observations_dropped( + self, mock_release, mock_series, mock_obs, fred_source, freeze_today + ): + freeze_today(date(2026, 3, 18)) + mock_release.return_value = make_fred_api_release() + mock_series.return_value = [make_fred_api_series()] + mock_obs.side_effect = [ + [{"id": "GOOD", "date": "2026-03-16", "value": 1.0}], + None, # BAD has no observations + ] + + dff = fred_source.fetch(dfq=None) + + assert len(dff) == 1 + assert dff.iloc[0]["id"] == "GOOD" + + @patch.object(FredSource, "questions", [{"id": "DGS10", "series_name": "10-year Treasury"}]) + @patch.object(FredSource, "_fetch_observations") + @patch.object(FredSource, "_fetch_series_info") + @patch.object(FredSource, "_fetch_release") + def test_bank_only_questions_included( + self, mock_release, mock_series, mock_obs, fred_source, freeze_today + ): + freeze_today(date(2026, 3, 18)) + mock_release.return_value = make_fred_api_release() + mock_series.return_value = [make_fred_api_series()] + mock_obs.return_value = [{"id": "DGS10", "date": "2026-03-16", "value": 4.30}] + dfq = make_question_df( + [ + {"id": "DGS10", "question": "Will DGS10 increase?"}, + {"id": "OLD_SERIES", "question": "Will OLD increase?"}, + ] + ) + + dff = fred_source.fetch(dfq=dfq) + + assert "OLD_SERIES" in dff["id"].values + + @patch.object(FredSource, "questions", [{"id": "DGS10", "series_name": "10-year Treasury"}]) + @patch.object(FredSource, "_fetch_observations") + @patch.object(FredSource, "_fetch_series_info") + @patch.object(FredSource, "_fetch_release") + def test_nullified_ids_excluded( + self, mock_release, mock_series, mock_obs, fred_source, freeze_today + ): + freeze_today(date(2026, 3, 18)) + mock_release.return_value = make_fred_api_release() + mock_series.return_value = [make_fred_api_series()] + mock_obs.return_value = [{"id": "DGS10", "date": "2026-03-16", "value": 4.30}] + dfq = make_question_df([{"id": "DGS10"}, {"id": "AMERIBOR"}]) # AMERIBOR nullified + + dff = fred_source.fetch(dfq=dfq) + + assert "AMERIBOR" not in dff["id"].values + + def test_api_key_required(self): + src = FredSource() + with pytest.raises(RuntimeError, match="api_key must be set"): + src.fetch() + + +# --------------------------------------------------------------------------- +# update() (pure data transformation) +# --------------------------------------------------------------------------- + + +class TestUpdate: + """Tests for FredSource.update.""" + + def test_basic_update(self, fred_source): + dfq = make_question_df([{"id": "EXISTING"}]) + dff = make_fred_fetch_df([{"id": "DGS10"}]) + + result = fred_source.update(dfq, dff) + + assert "DGS10" in result.dfq["id"].values + assert "DGS10" in result.resolution_files + QuestionFrame.validate(result.dfq) + + def test_new_question_inserted(self, fred_source): + dfq = make_question_df([{"id": "EXISTING"}]) + dff = make_fred_fetch_df([{"id": "NEW_SERIES"}]) + + result = fred_source.update(dfq, dff) + + assert len(result.dfq) == 2 + assert set(result.dfq["id"].tolist()) == {"EXISTING", "NEW_SERIES"} + + def test_existing_question_updated(self, fred_source): + dfq = make_question_df([{"id": "DGS10", "question": "Old text"}]) + dff = make_fred_fetch_df([{"id": "DGS10", "question": "New text"}]) + + result = fred_source.update(dfq, dff) + + assert len(result.dfq) == 1 + assert result.dfq.iloc[0]["question"] == "New text" + + def test_resolution_files_extracted(self, fred_source): + resolutions = [ + {"id": "X", "date": "2026-03-14", "value": 1.0}, + {"id": "X", "date": "2026-03-15", "value": 1.5}, + ] + dff = make_fred_fetch_df([{"id": "X", "resolutions": resolutions}]) + dfq = make_question_df([{"id": "placeholder"}]).iloc[:0] + + result = fred_source.update(dfq, dff) + + assert "X" in result.resolution_files + df_res = result.resolution_files["X"] + assert len(df_res) == 2 + assert list(df_res.columns) == ["id", "date", "value"] + + def test_transient_fields_stripped(self, fred_source): + dfq = make_question_df([{"id": "placeholder"}]).iloc[:0] + dff = make_fred_fetch_df([{"id": "DGS10"}]) + + result = fred_source.update(dfq, dff) + + for col in ["fetch_datetime", "probability", "resolutions"]: + assert col not in result.dfq.columns + + def test_nullified_ids_dropped_from_dfq(self, fred_source): + dfq = make_question_df([{"id": "AMERIBOR"}, {"id": "CURRCIR"}, {"id": "DGS10"}]) + dff = make_fred_fetch_df([{"id": "DGS10"}]) - def test_update_questions_drops_nullified_currcir_from_dfq(self): - dfq = make_question_df([{"id": "CURRCIR"}, {"id": "AAA10Y"}]) + result = fred_source.update(dfq, dff) - result = update_questions(dfq, pd.DataFrame()) + assert "AMERIBOR" not in result.dfq["id"].values + assert "CURRCIR" not in result.dfq["id"].values - assert "CURRCIR" not in result["id"].values - assert "AAA10Y" in result["id"].values + def test_update_works_without_api_key(self): + src = FredSource() # no api_key set + dfq = make_question_df([{"id": "placeholder"}]).iloc[:0] + dff = make_fred_fetch_df([{"id": "DGS10"}]) + result = src.update(dfq, dff) + assert "DGS10" in result.dfq["id"].values From ab682c8774b887a5f8da50f5fe653901681d2868 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 15 Jun 2026 10:54:04 +0300 Subject: [PATCH 2/2] fix: rename fred_questions files so it's next to fred.py --- src/sources/_metadata.py | 2 +- src/sources/{_fred_questions.py => fred_questions.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/sources/{_fred_questions.py => fred_questions.py} (100%) diff --git a/src/sources/_metadata.py b/src/sources/_metadata.py index d80e572e..aabc9b86 100644 --- a/src/sources/_metadata.py +++ b/src/sources/_metadata.py @@ -9,7 +9,7 @@ from _fb_types import NullifiedQuestion, SourceType from helpers.constants import BENCHMARK_START_DATE_DATETIME_DATE -from ._fred_questions import FRED_QUESTIONS +from .fred_questions import FRED_QUESTIONS SOURCE_METADATA = { "acled": { diff --git a/src/sources/_fred_questions.py b/src/sources/fred_questions.py similarity index 100% rename from src/sources/_fred_questions.py rename to src/sources/fred_questions.py