From 23c42c97187cb762d4af500cf3b796a573c76027 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 13:20:03 -0700 Subject: [PATCH 01/10] add time interp --- imap_processing/ialirt/l0/process_swe.py | 27 +++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index ea88e839e..c174476ac 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -157,7 +157,7 @@ def get_ialirt_energies() -> list: return energy -def normalize_counts(counts: NDArray, latest_cal: pd.Series) -> NDArray: +def normalize_counts(counts: NDArray, interp_cal: NDArray) -> NDArray: """ Normalize the counts using the latest calibration factor. @@ -165,18 +165,17 @@ def normalize_counts(counts: NDArray, latest_cal: pd.Series) -> NDArray: ---------- counts : np.ndarray Array of counts. - latest_cal : pd.Series - Array of latest calibration factors. + interp_cal : np.ndarray + Array of calibration factors. Returns ------- norm_counts : np.ndarray Array of normalized counts. """ - latest_cal = latest_cal.to_numpy() # Norm counts where counts are non-negative - norm_counts = counts * (latest_cal / GEOMETRIC_FACTORS)[:, np.newaxis] + norm_counts = counts * (interp_cal / GEOMETRIC_FACTORS)[:, np.newaxis] norm_counts[norm_counts < 0] = 0 return norm_counts @@ -522,10 +521,22 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list # Grab the latest calibration factor in_flight_cal_df = read_in_flight_cal_data(in_flight_cal_files) - latest_cal = in_flight_cal_df.sort_values("met_time").iloc[-1][1::] + group_time = int( + grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] + ) + cal_cols = [f"cem{i}" for i in range(1, 8)] + cal_met = in_flight_cal_df["met_time"].to_numpy() + + interp_cal = np.array( + [ + float(np.interp(group_time, cal_met, in_flight_cal_df[cem].to_numpy())) + for cem in cal_cols + ], + dtype=np.float64, + ) - normalized_first_half = normalize_counts(corrected_first_half, latest_cal) - normalized_second_half = normalize_counts(corrected_second_half, latest_cal) + normalized_first_half = normalize_counts(corrected_first_half, interp_cal) + normalized_second_half = normalize_counts(corrected_second_half, interp_cal) # Sum over the 7 detectors summed_first_half_cem = np.sum(normalized_first_half, axis=1) From 8dbdb78978a852428ba928649d6591710dfc4ded Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 20:43:42 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/ialirt/l0/process_swe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index c174476ac..c2bd07304 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -3,7 +3,6 @@ import logging import numpy as np -import pandas as pd import xarray as xr from numpy.typing import NDArray @@ -173,7 +172,6 @@ def normalize_counts(counts: NDArray, interp_cal: NDArray) -> NDArray: norm_counts : np.ndarray Array of normalized counts. """ - # Norm counts where counts are non-negative norm_counts = counts * (interp_cal / GEOMETRIC_FACTORS)[:, np.newaxis] norm_counts[norm_counts < 0] = 0 From aa309e7ec21ef8226aca7f23aeaa734c627f1540 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 13:54:59 -0700 Subject: [PATCH 03/10] update tests --- .../tests/ialirt/unit/test_process_swe.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_swe.py b/imap_processing/tests/ialirt/unit/test_process_swe.py index 1df92c3fb..36b280c52 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swe.py +++ b/imap_processing/tests/ialirt/unit/test_process_swe.py @@ -324,19 +324,7 @@ def test_normalize_counts(): dtype=np.uint8, ) - latest_cal = pd.Series( - [2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0], - index=[ - "cem1", - "cem2", - "cem3", - "cem4", - "cem5", - "cem6", - "cem7", - ], # Simulating real data structure - dtype=np.float64, - ) + latest_cal = np.array([2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]) expected = np.zeros((2, 7, 3), dtype=np.float64) for i in range(2): From d2aab1f6278ff22f881a85936374a64d60cadf41 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 14:11:17 -0700 Subject: [PATCH 04/10] add in swe epoch --- imap_processing/ialirt/l0/process_swe.py | 38 +++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index c2bd07304..053a2e6fa 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -519,22 +519,44 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list # Grab the latest calibration factor in_flight_cal_df = read_in_flight_cal_data(in_flight_cal_files) - group_time = int( - grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] - ) cal_cols = [f"cem{i}" for i in range(1, 8)] cal_met = in_flight_cal_df["met_time"].to_numpy() - interp_cal = np.array( + group_time_first_half = int( + grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] + ) + interp_cal_first_half = np.array( [ - float(np.interp(group_time, cal_met, in_flight_cal_df[cem].to_numpy())) + float( + np.interp( + group_time_first_half, cal_met, in_flight_cal_df[cem].to_numpy() + ) + ) + for cem in cal_cols + ], + dtype=np.float64, + ) + group_time_second_half = int( + grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] + ) + interp_cal_second_half = np.array( + [ + float( + np.interp( + group_time_first_half, cal_met, in_flight_cal_df[cem].to_numpy() + ) + ) for cem in cal_cols ], dtype=np.float64, ) - normalized_first_half = normalize_counts(corrected_first_half, interp_cal) - normalized_second_half = normalize_counts(corrected_second_half, interp_cal) + normalized_first_half = normalize_counts( + corrected_first_half, interp_cal_first_half + ) + normalized_second_half = normalize_counts( + corrected_second_half, interp_cal_second_half + ) # Sum over the 7 detectors summed_first_half_cem = np.sum(normalized_first_half, axis=1) @@ -568,6 +590,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_first_half) | { "instrument": "swe", + "swe_epoch": group_time_first_half, "swe_normalized_counts": [int(val) for val in summed_first], "swe_counterstreaming_electrons": bde_first_half, }, @@ -576,6 +599,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_second_half) | { "instrument": "swe", + "swe_epoch": group_time_second_half, "swe_normalized_counts": [int(val) for val in summed_second], "swe_counterstreaming_electrons": bde_second_half, }, From a16ba6b282c2e59a1e7abc145ef20de08fb1235f Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 14:44:47 -0700 Subject: [PATCH 05/10] add midpoint --- imap_processing/ialirt/l0/process_swe.py | 28 +++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 053a2e6fa..fd26317ed 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -522,28 +522,40 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list cal_cols = [f"cem{i}" for i in range(1, 8)] cal_met = in_flight_cal_df["met_time"].to_numpy() - group_time_first_half = int( - grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] + group_time_first_half = ( + grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values ) + group_time_first_half_mid = ( + group_time_first_half[0] + group_time_first_half[-1] + ) // 2 + interp_cal_first_half = np.array( [ float( np.interp( - group_time_first_half, cal_met, in_flight_cal_df[cem].to_numpy() + int(group_time_first_half[0]), + cal_met, + in_flight_cal_df[cem].to_numpy(), ) ) for cem in cal_cols ], dtype=np.float64, ) - group_time_second_half = int( - grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values[0] + group_time_second_half = ( + grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values ) + group_time_second_half_mid = ( + group_time_first_half[0] + group_time_first_half[-1] + ) // 2 + interp_cal_second_half = np.array( [ float( np.interp( - group_time_first_half, cal_met, in_flight_cal_df[cem].to_numpy() + int(group_time_second_half[0]), + cal_met, + in_flight_cal_df[cem].to_numpy(), ) ) for cem in cal_cols @@ -590,7 +602,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_first_half) | { "instrument": "swe", - "swe_epoch": group_time_first_half, + "swe_epoch": int(group_time_first_half_mid), "swe_normalized_counts": [int(val) for val in summed_first], "swe_counterstreaming_electrons": bde_first_half, }, @@ -599,7 +611,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_second_half) | { "instrument": "swe", - "swe_epoch": group_time_second_half, + "swe_epoch": int(group_time_second_half_mid), "swe_normalized_counts": [int(val) for val in summed_second], "swe_counterstreaming_electrons": bde_second_half, }, From 928cf90202053ac65c9333e872180fdd5e1f281d Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 15:03:19 -0700 Subject: [PATCH 06/10] improve grouping --- imap_processing/ialirt/l0/process_swe.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index fd26317ed..659f6accf 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -523,7 +523,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list cal_met = in_flight_cal_df["met_time"].to_numpy() group_time_first_half = ( - grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values + grouped["time_seconds"].where(grouped["swe_seq"] < 30, drop=True).values ) group_time_first_half_mid = ( group_time_first_half[0] + group_time_first_half[-1] @@ -533,7 +533,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list [ float( np.interp( - int(group_time_first_half[0]), + int(group_time_first_half_mid), cal_met, in_flight_cal_df[cem].to_numpy(), ) @@ -543,17 +543,17 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list dtype=np.float64, ) group_time_second_half = ( - grouped["time_seconds"].where(grouped["swe_seq"] == 0, drop=True).values + grouped["time_seconds"].where(grouped["swe_seq"] >= 30, drop=True).values ) group_time_second_half_mid = ( - group_time_first_half[0] + group_time_first_half[-1] + group_time_second_half[0] + group_time_second_half[-1] ) // 2 interp_cal_second_half = np.array( [ float( np.interp( - int(group_time_second_half[0]), + int(group_time_second_half_mid), cal_met, in_flight_cal_df[cem].to_numpy(), ) From e7d03f20f01e22accf73afa3bfe64a6929f072d4 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 19 Jan 2026 15:37:44 -0700 Subject: [PATCH 07/10] convert time --- imap_processing/ialirt/l0/process_swe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 659f6accf..37d6ddae4 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -602,7 +602,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_first_half) | { "instrument": "swe", - "swe_epoch": int(group_time_first_half_mid), + "swe_epoch": int(met_to_ttj2000ns(group_time_first_half_mid)), "swe_normalized_counts": [int(val) for val in summed_first], "swe_counterstreaming_electrons": bde_first_half, }, @@ -611,7 +611,7 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list _populate_instrument_header_items(met_second_half) | { "instrument": "swe", - "swe_epoch": int(group_time_second_half_mid), + "swe_epoch": int(met_to_ttj2000ns(group_time_second_half_mid)), "swe_normalized_counts": [int(val) for val in summed_second], "swe_counterstreaming_electrons": bde_second_half, }, From f0e44f94b1945c3b8104eb8b464295cf1dfdd414 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 20 Jan 2026 13:21:45 -0700 Subject: [PATCH 08/10] fix test --- imap_processing/tests/ialirt/unit/test_process_swe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_swe.py b/imap_processing/tests/ialirt/unit/test_process_swe.py index 36b280c52..a3302966c 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swe.py +++ b/imap_processing/tests/ialirt/unit/test_process_swe.py @@ -177,7 +177,7 @@ def test_process_spacecraft_packet( swe_product = process_swe(sc_xarray_data, [in_flight_cal_file]) - assert len(swe_product[0].keys()) == 7 + assert len(swe_product[0].keys()) == 8 def test_get_energy(): From 095f38ee6387f4c9c6ea48a0ee0cbe731787153d Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 22 Jan 2026 14:54:41 -0700 Subject: [PATCH 09/10] pr response --- imap_processing/ialirt/l0/process_swe.py | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 37d6ddae4..68db03119 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -517,11 +517,13 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list corrected_first_half = deadtime_correction(counts_first_half, 80 * 10**3) corrected_second_half = deadtime_correction(counts_second_half, 80 * 10**3) - # Grab the latest calibration factor + # Interpolate to find the correct calibration factor in_flight_cal_df = read_in_flight_cal_data(in_flight_cal_files) + # Get names of all 7 cems in the calibration file cal_cols = [f"cem{i}" for i in range(1, 8)] cal_met = in_flight_cal_df["met_time"].to_numpy() + # Find the middle timestamp of the first group group_time_first_half = ( grouped["time_seconds"].where(grouped["swe_seq"] < 30, drop=True).values ) @@ -529,34 +531,32 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list group_time_first_half[0] + group_time_first_half[-1] ) // 2 + # Interpolate to the appropriate calibration factor interp_cal_first_half = np.array( [ - float( - np.interp( - int(group_time_first_half_mid), - cal_met, - in_flight_cal_df[cem].to_numpy(), - ) + np.interp( + int(group_time_first_half_mid), + cal_met, + in_flight_cal_df[cem].to_numpy(), ) for cem in cal_cols ], dtype=np.float64, ) + # Find the middle timestamp of the second group group_time_second_half = ( grouped["time_seconds"].where(grouped["swe_seq"] >= 30, drop=True).values ) group_time_second_half_mid = ( group_time_second_half[0] + group_time_second_half[-1] ) // 2 - + # Interpolate to the appropriate calibration factor interp_cal_second_half = np.array( [ - float( - np.interp( - int(group_time_second_half_mid), - cal_met, - in_flight_cal_df[cem].to_numpy(), - ) + np.interp( + int(group_time_second_half_mid), + cal_met, + in_flight_cal_df[cem].to_numpy(), ) for cem in cal_cols ], From 63b642809829b9b5a8976985057aff3a3c4c9e30 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 22 Jan 2026 14:58:06 -0700 Subject: [PATCH 10/10] add import --- imap_processing/ialirt/l0/process_swe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 68db03119..7389f28f7 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -11,6 +11,7 @@ find_groups, ) from imap_processing.ialirt.utils.time import calculate_time +from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.swe.l1a.swe_science import decompressed_counts from imap_processing.swe.l1b.swe_l1b import ( deadtime_correction,