v2.0.0: updates for required authentication

.gitignore

@@ -4,3 +4,4 @@ build/
 *.egg-info/
 __pycache__/
 .pyc
+.env

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "neonutilities"
-version = "1.2.2"
+version = "2.0.0"
 authors = [
     {name="Claire Lunch", email="clunch@battelleecology.org"},
     {name="Bridget Hass", email="bhass@battelleecology.org"},
@@ -15,6 +15,7 @@ description="A package for accessing and wrangling data generated and published
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "duckdb",
     "importlib-resources",
     "pandas",
     "parameterized",

requirements.txt

@@ -1,3 +1,4 @@
+duckdb>=1.5.0
 importlib_resources>=6.1.1
 pandas>=2.1.4
 parameterized>=0.9.0

src/neonutilities/__resources__/shared_aquatic.csv

@@ -1,37 +1,61 @@
 site,towerSite,product
+HOPB,HARV,DP1.00044.001
+HOPB,HARV,DP1.00045.001
 HOPB,HARV,DP1.00006.001
 HOPB,HARV,DP1.00038.001
 HOPB,HARV,DP1.00013.001
+POSE,SCBI,DP1.00044.001
+POSE,SCBI,DP1.00045.001
 POSE,SCBI,DP1.00006.001
 POSE,SCBI,DP1.00038.001
 POSE,SCBI,DP1.00013.001
+LEWI,BLAN,DP1.00044.001
+LEWI,BLAN,DP1.00045.001
 LEWI,BLAN,DP1.00006.001
 LEWI,BLAN,DP1.00038.001
 LEWI,BLAN,DP1.00013.001
+BARC,OSBS,DP1.00044.001
+BARC,OSBS,DP1.00045.001
 BARC,OSBS,DP1.00006.001
 BARC,OSBS,DP1.00038.001
 BARC,OSBS,DP1.00013.001
+SUGG,OSBS,DP1.00044.001
+SUGG,OSBS,DP1.00045.001
 SUGG,OSBS,DP1.00006.001
 SUGG,OSBS,DP1.00038.001
 SUGG,OSBS,DP1.00013.001
+FLNT,JERC,DP1.00044.001
+FLNT,JERC,DP1.00045.001
 FLNT,JERC,DP1.00006.001
 FLNT,JERC,DP1.00038.001
 FLNT,JERC,DP1.00013.001
+CRAM,UNDE,DP1.00044.001
+CRAM,UNDE,DP1.00045.001
 CRAM,UNDE,DP1.00006.001
 CRAM,UNDE,DP1.00038.001
 CRAM,UNDE,DP1.00013.001
+LIRO,UNDE,DP1.00044.001
+LIRO,UNDE,DP1.00045.001
 LIRO,UNDE,DP1.00006.001
 LIRO,UNDE,DP1.00038.001
 LIRO,UNDE,DP1.00013.001
+KING,KONZ,DP1.00044.001
+KING,KONZ,DP1.00045.001
 KING,KONZ,DP1.00006.001
 KING,KONZ,DP1.00038.001
 KING,KONZ,DP1.00013.001
+WALK,ORNL,DP1.00044.001
+WALK,ORNL,DP1.00045.001
 WALK,ORNL,DP1.00006.001
 WALK,ORNL,DP1.00038.001
 WALK,ORNL,DP1.00013.001
+LECO,GRSM,DP1.00044.001
+LECO,GRSM,DP1.00045.001
 LECO,GRSM,DP1.00006.001
 LECO,GRSM,DP1.00038.001
 LECO,GRSM,DP1.00013.001
+MAYF,TALL,DP1.00044.001
+MAYF,TALL,DP1.00045.001
 MAYF,TALL,DP1.00006.001
 MAYF,TALL,DP1.00038.001
 MAYF,TALL,DP1.00013.001
@@ -41,6 +65,8 @@ BLWA,DELA,DP1.00024.001
 BLWA,DELA,DP1.00098.001
 BLWA,DELA,DP1.00002.001
 BLWA,DELA,DP1.00023.001
+BLWA,DELA,DP1.00044.001
+BLWA,DELA,DP1.00045.001
 BLWA,DELA,DP1.00006.001
 BLWA,DELA,DP1.00038.001
 BLWA,DELA,DP1.00013.001
@@ -50,30 +76,48 @@ TOMB,LENO,DP1.00024.001
 TOMB,LENO,DP1.00098.001
 TOMB,LENO,DP1.00002.001
 TOMB,LENO,DP1.00023.001
+TOMB,LENO,DP1.00044.001
+TOMB,LENO,DP1.00045.001
 TOMB,LENO,DP1.00006.001
 TOMB,LENO,DP1.00038.001
 TOMB,LENO,DP1.00013.001
+PRPO,WOOD,DP1.00044.001
+PRPO,WOOD,DP1.00045.001
 PRPO,WOOD,DP1.00006.001
 PRPO,WOOD,DP1.00038.001
 PRPO,WOOD,DP1.00013.001
+PRLA,DCFS,DP1.00044.001
+PRLA,DCFS,DP1.00045.001
 PRLA,DCFS,DP1.00006.001
 PRLA,DCFS,DP1.00038.001
 PRLA,DCFS,DP1.00013.001
+BLDE,YELL,DP1.00044.001
+BLDE,YELL,DP1.00045.001
 BLDE,YELL,DP1.00006.001
 BLDE,YELL,DP1.00038.001
 BLDE,YELL,DP1.00013.001
+COMO,NIWO,DP1.00044.001
+COMO,NIWO,DP1.00045.001
 COMO,NIWO,DP1.00006.001
 COMO,NIWO,DP1.00038.001
 COMO,NIWO,DP1.00013.001
+MART,WREF,DP1.00044.001
+MART,WREF,DP1.00045.001
 MART,WREF,DP1.00006.001
 MART,WREF,DP1.00038.001
 MART,WREF,DP1.00013.001
+TECR,TEAK,DP1.00044.001
+TECR,TEAK,DP1.00045.001
 TECR,TEAK,DP1.00006.001
 TECR,TEAK,DP1.00038.001
 TECR,TEAK,DP1.00013.001
+OKSR,TOOL,DP1.00044.001
+OKSR,TOOL,DP1.00045.001
 OKSR,TOOL,DP1.00006.001
 OKSR,TOOL,DP1.00038.001
 OKSR,TOOL,DP1.00013.001
+CARI,BONA,DP1.00044.001
+CARI,BONA,DP1.00045.001
 CARI,BONA,DP1.00006.001
 CARI,BONA,DP1.00038.001
 CARI,BONA,DP1.00013.001

src/neonutilities/aop_download.py

@@ -31,14 +31,14 @@
 import re
 
 # import time
-from time import sleep
 from tqdm import tqdm
 
 # local imports
 from . import __resources__
 from .helper_mods.api_helpers import get_api
-from .helper_mods.api_helpers import token_check
+from .helper_mods.api_helpers import token_check, auth_check
 from .helper_mods.api_helpers import download_file, calculate_crc32c
+from .helper_mods.api_helpers import baseurl
 from .helper_mods.metadata_helpers import convert_byte_size
 from .get_issue_log import get_issue_log
 from .citation import get_citation
@@ -221,7 +221,7 @@ def get_neon_sites():
 
 
 def get_data_product_name(dpid):
-    dpid_api_response = get_api(f"https://data.neonscience.org/api/v0/products/{dpid}")
+    dpid_api_response = get_api(f"{baseurl}products/{dpid}")
     product_name = dpid_api_response.json()["data"]["productName"]
     return product_name
 
@@ -307,7 +307,7 @@ def get_aop_dpids():
     >>> active_dpids = get_active_dpids()
     # This will return a list of all active NEON data product IDs.
     """
-    response = get_api("https://data.neonscience.org/api/v0/products")
+    response = get_api(baseurl + "products")
 
     response_dict = response.json()
     # all_neon_dpids = [item["productCode"] for item in response_dict["data"]]
@@ -522,7 +522,7 @@ def list_available_dates(dpid, site):
         >>> list_available_dates('DP1.10098.001','HOPB')
         ValueError: There are no data available for the data product DP1.10098.001 at the site HOPB.
     """
-    product_url = "https://data.neonscience.org/api/v0/products/" + dpid
+    product_url = baseurl + "products/" + dpid
     response = get_api(api_url=product_url)  # add input for token?
 
     # raise value error and print message if dpid isn't formatted as expected
@@ -692,7 +692,7 @@ def get_aop_tile_extents(dpid, site, year, token=None):
         token = None
 
     # query the products endpoint for the product requested
-    response = get_api("https://data.neonscience.org/api/v0/products/" + dpid, token)
+    response = get_api(baseurl + "products/" + dpid, token)
 
     # exit function if response is None (eg. if no internet connection)
     if response is None:
@@ -880,8 +880,13 @@ def by_file_aop(
     if token is not None:
         token = token_check(token)
 
+    # check for access to the data query endpoint to test whether token is valid
+    authcheck = auth_check(token=token)
+    if authcheck is None:
+        pass
+
     # query the products endpoint for the product requested
-    response = get_api("https://data.neonscience.org/api/v0/products/" + dpid, token)
+    response = get_api(baseurl + "products/" + dpid, token)
 
     # exit function if response is None (eg. if no internet connection)
     if response is None:
@@ -1170,9 +1175,11 @@ def by_file_aop(
                 logging.info("Skipped overwriting files with mismatched checksums.")
 
     else:
+        tstart = datetime.now()
         for file in tqdm(files):
             download_file(
-                url=file, savepath=download_path, chunk_size=chunk_size, token=token
+                url=file, savepath=download_path, chunk_size=chunk_size, 
+                token=token, tstart=tstart
             )
 
     # download issue log table
@@ -1404,8 +1411,13 @@ def by_tile_aop(
     if token is not None:
         token = token_check(token)
 
+    # check for access to the data query endpoint to test whether token is valid
+    authcheck = auth_check(token=token)
+    if authcheck is None:
+        pass
+
     # query the products endpoint for the product requested
-    response = get_api("https://data.neonscience.org/api/v0/products/" + dpid, token)
+    response = get_api(baseurl + "products/" + dpid, token)
 
     # exit function if response is None (eg. if no internet connection)
     if response is None:
@@ -1685,6 +1697,7 @@ def get_buffer_coords(easting, northing, buffer):
                 logging.info(f"  {f}")
 
             # Download files that do not exist locally
+            tstart = datetime.now()
             for _, row in tqdm(
                 files_to_download.iterrows(), total=len(files_to_download)
             ):
@@ -1693,6 +1706,7 @@ def get_buffer_coords(easting, northing, buffer):
                     savepath=download_path,
                     chunk_size=chunk_size,
                     token=token,
+                    tstart=tstart
                 )
 
             if not files_to_skip.empty:
@@ -1750,6 +1764,7 @@ def get_buffer_coords(easting, northing, buffer):
 
             if response.lower() == "y":
                 logging.info("Overwriting these files with the latest available data.")
+                tstart = datetime.now()
                 for idx, row in tqdm(
                     mismatched_files.iterrows(), total=len(mismatched_files)
                 ):
@@ -1758,13 +1773,16 @@ def get_buffer_coords(easting, northing, buffer):
                         savepath=download_path,
                         chunk_size=chunk_size,
                         token=token,
+                        tstart=tstart
                     )
             else:
                 logging.info("Skipped overwriting files with mismatched checksums.")
     else:  # if skip_if_exists=False (default behavior)
+        tstart = datetime.now()
         for file in tqdm(files):
             download_file(
-                url=file, savepath=download_path, chunk_size=chunk_size, token=token
+                url=file, savepath=download_path, chunk_size=chunk_size, token=token,
+                tstart=tstart
             )
 
     # download issue log table

src/neonutilities/citation.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
 import requests
+import logging
 
 
 def get_citation(dpid, release):
@@ -55,7 +56,7 @@ def get_citation(dpid, release):
         relinfo = next((i for i in rels if i["release"] == release), None)
 
         if relinfo is None:
-            print("There are no data with dpid=" + dpid + " and release=" + release)
+            logging.info("No citation found for dpid=" + dpid + " and release=" + release)
             return relinfo
 
         else:

src/neonutilities/get_issue_log.py

@@ -17,6 +17,7 @@
 import logging
 import pandas as pd
 from .helper_mods.api_helpers import get_api
+from .helper_mods.api_helpers import baseurl
 
 logging.basicConfig(level=logging.INFO, format="%(message)s")
 
@@ -58,7 +59,7 @@ def get_change_log_df(dpid, token=None):
         'issue', 'resolution'
     """
     req = get_api(
-        api_url=f"https://data.neonscience.org/api/v0/products/{dpid}", token=token
+        api_url=f"{baseurl}products/{dpid}", token=token
     )
     if req is None:
         logging.info(f"Error in metadata retrieval for {dpid}. Issue log not found.")