From 8dc1737cd29fa1c228df748283f08d8433d56029 Mon Sep 17 00:00:00 2001 From: Ian Black Date: Wed, 18 Jun 2025 09:42:35 -0700 Subject: [PATCH 1/3] Update .gitignore for PyCharm. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 62567ab..f97d6bd 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,9 @@ ENV/ # vscode settings.json + +# PyCharm +.idea/ + +# Custom +scratch/ \ No newline at end of file From 9253f1712a0e439f4e4ffb7bad93de93859f3674 Mon Sep 17 00:00:00 2001 From: Ian Black Date: Wed, 18 Jun 2025 09:43:04 -0700 Subject: [PATCH 2/3] Adds different .hdr files for testing. --- .../test_files/D20190929T185847_IFCB122.hdr | 134 ++++++++++++++++++ .../test_files/D20230521T210245_IFCB174.hdr | 132 +++++++++++++++++ .../test_files/D20231021T154635_IFCB199.hdr | 132 +++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 ifcb/tests/test_files/D20190929T185847_IFCB122.hdr create mode 100644 ifcb/tests/test_files/D20230521T210245_IFCB174.hdr create mode 100644 ifcb/tests/test_files/D20231021T154635_IFCB199.hdr diff --git a/ifcb/tests/test_files/D20190929T185847_IFCB122.hdr b/ifcb/tests/test_files/D20190929T185847_IFCB122.hdr new file mode 100644 index 0000000..657aac9 --- /dev/null +++ b/ifcb/tests/test_files/D20190929T185847_IFCB122.hdr @@ -0,0 +1,134 @@ +softwareVersion: Imaging FlowCytobot Acquisition Software version 1.1.5.19 +HousekeepingFirmwareVersion: 32 +AnalogFirmwareVersion: 45 +SyringeNumber: 1 +sampleTime: 2019-09-29T18:58:47Z +imagerID: 122 +KloehnPort: COM3 +DataFilePath: C:\Data\Kavanaugh_Shimada_Sep2019\ +syringeOffset: 5437 +appDebug: 1 +viewImages: 0 +blobAnalysis: 1 +pumpDriveBoardPresent: 0 +backflushWithSample: 0 +outputFiles: 1 +fluidicsVerbose: 0 +fluidsActive: 1 +binarizeThreshold: 5 +blobXgrowAmount: 20 +imageResizeFactor: 8 +minimumBlobArea: 1500 +triggerPulseEndtoADCstart_x2170ns_DAQ_MCC_only: 8 +PMTAhighVoltage: 0.550000 +PMTBhighVoltage: 0.700000 +PMTChighVoltage: 0.000000 +DAQ_MCCserialPort_DAC_MCConly: COM1 +ADCdataRateDAQ_MCConly: 240 +FlashlampControlVoltage: 2.400000 +FanTemperatureThreshold_DAC_MCConly: 117964 +FanTemperatureHysteresis_DAC_MCConly: 123 +ValveHumidityThreshold_DAC_MCConly: 65621 +ValveHumidityHysteresis_DAC_MCConly: 567 +ValveHoldTime_x100us_DAC_MCConly: 781 +HKTRIGGERtoFlashlampDelayTime_x434ns_DAC_MCConly: 205 +FlashlampPulseLength_x434ns_DAC_MCConly: 34 +CameraPulseLength_x434ns_DAC_MCConly: 505 +SyringeSampleVolume: 5.000000 +blobYgrowAmount: 5 +minimumGapBewtweenAdjacentBlobs: 50 +focusMotorSmallStep_ms: 25 +focusMotorLargeStep_ms: 200 +PMTAtriggerThreshold_DAQ_MCConly: 0.150000 +PMTBtriggerThreshold_DAQ_MCConly: 0.150000 +PMTCtriggerThreshold_DAQ_MCConly: 0.000000 +PMTDtriggerThreshold_DAQ_MCConly: 0.000000 +PMTtriggerSelection_DAQ_MCConly: 3 +ADCPGA_DAQ_MCConly: 1 +triggerAckTimeout_x34720ns_DAQ_MCConly: 21600 +pulseStretchTime_x2170ns_DAQ_MCConly: 18 +debubbleWithSample: 0 +pumpDriveVoltage: 24.000000 +autoWindowsShutdown: 0 +NumberSyringesToAutoRun: 3 +intervalBetweenAlternateHardwareSettings: 0 +altPMTATriggerThreshold_DAQ_MCConly: 0.000000 +altPMTBTriggerThreshold_DAQ_MCConly: 0.000000 +altPMTCTriggerThreshold_DAQ_MCConly: 0.000000 +altPMTDTriggerThreshold_DAQ_MCConly: 0.000000 +altPMTAHighVoltage: 0.000000 +altPMTBHighVoltage: 0.000000 +altPMTCHighVoltage: 0.000000 +altADCPGA_DAQ_MCConly: 0 +altPMTtriggerSelection_DAQ_MCConly: 0 +NumberSyringesBetweenBeadsRun: 0 +laserMotorSmallStep_ms: 500 +laserMotorLargeStep_ms: 50000 +altSyringeSampleVolume: 0.100000 +maximumPulseLength_x2170ns_DAQ_MCConly: 13823 +expertTabVisible: 1 +TCPport: 12345 +HumidityAlarmThreshold(%): 80.000000 +PumpStates: 2 +BeadsSampleVolume: 2.500000 +RunFastFactor: 10.000000 +TimeoutForFirstFrame: 90000 +sampleVolume2skip: 0.000000 +altSampleVolume2skip: 0.000000 +altSamplePortNumber: 8 +primeSampleTube: 0 +stainSample_manual: 0 +StainAuto_auto: 0 +Alt_FlashlampControlVoltage: 0.000000 +stainTime: 30 +rinseOption: 0 +primaryIntakePort: 8 +secondaryIntakePort: 8 +normSamplePortNumber: 8 +replaceFromSecondary: 0 +syringeSamplingSpeed: 716963 +baseSyringeSamplingSpeed: 8962 +runSampleFast: 1 +laserState: 65537 +fill2bypassFilterState: 0 +fill3newSheathState: 0 +LockFilterSlider: 0 +syringeSize: 5 +hasCamera: 1 +AcousticFocusingOnALT: 0 +RefillAfterDebubble: 0 +FileComment: UNDERWAY +PZTvolts: 40 +PZTfrequency: 1772000 +TemperatureAdjust: 0 +TemperatureAdjust: 0 +FreqUpdateInterval: 3 +DoFreqScan: 0 +StartFreq: 1772000.000000 +FreqStep: 500.000000 +EndFreq: 1800000.000000 +StepDuration: 15000.000000 +stepCount: 1 +Tadjm: 2493.000000 +Tadjb: 1690500.000000 +Tadjm2: 0.000000 +beadSampleTotalVolume: 2.400000 +backflushAfterSampleNoRefill: 0 +HasFastTriggerMod: 0 +BleachToExhaust: 0 +PrimeIntakeVolume: 5.000000 +PromptForFileComment: 0 +DebubbleNeedleVolume: 1.000000 +DebubbleConeVolume: 2.000000 +DebubbleRefillVolume: 3.500000 +DebubbleRefillVolumeInjected: 3.000000 +BypassFieldLimits: 0 +BleachRinseCount: 5 +HasCommandsWindow: 1 +EquipmentType: 1 +runType: NORMAL +runTime: 11.791851 +inhibitTime: 8.869473 +temperature: 27.160471 +humidity: 50.356638 +ADCFileFormat: trigger#, ADC_time, PMTA, PMTB, PMTC, PMTD, peakA, peakB, peakC, peakD, time of flight, grabtimestart, grabtimeend, ROIx, ROIy, ROIwidth, ROIheight, start_byte, comparator_out, STartPoint, SignalLength, status, runTime, inhibitTime diff --git a/ifcb/tests/test_files/D20230521T210245_IFCB174.hdr b/ifcb/tests/test_files/D20230521T210245_IFCB174.hdr new file mode 100644 index 0000000..da28e82 --- /dev/null +++ b/ifcb/tests/test_files/D20230521T210245_IFCB174.hdr @@ -0,0 +1,132 @@ +SoftwareVersion: 2.5.0.0 +AnalogFirmware: 45 +HousekeepingFirmware: 32 +sampleNumber: 1 +sampleType: Normal +triggerCount: 1815 +roiCount: 468 +humidity: 53.133032730601975 +temperature: 19.516250095368896 +runTime: 1200.8731944444444 +inhibitTime: 151.66454861111112 +ADCFileFormat: trigger#, ADCtime, PMTA, PMTB, PMTC, PMTD, PeakA, PeakB, PeakC, PeakD, TimeOfFlight, GrabTimeStart, GrabTimeEnd, RoiX, RoiY, RoiWidth, RoiHeight, StartByte, ComparatorOut, StartPoint, SignalLength, Status, RunTime, InhibitTime +DAQ_MCCserialPort_DAC_MCConly: /dev/ttyS2 +UvModulePresent: False +UvWithSample: False +auxPower1: False +autoStart: False +InteractiveAutoStart: False +autoShutdown: False +HumidityAlarmThreshold(%): 90 +ValveHumidityHysteresis_DAC_MCConly: 567 +FanTemperatureThreshold_DAC_MCConly: 30744 +FanTemperatureHysteresis_DAC_MCConly: 123 +valveDelay: 781 +FlashlampControlVoltage: 2.7999 +HKTRIGGERtoFlashlampDelayTime_x434ns_DAC_MCConly: 120 +FlashlampPulseLength_x434ns_DAC_MCConly: 20 +CameraPulseLength_x434ns_DAC_MCConly: 500 +cameraGain: 6 +binarizeThreshold: 6 +minimumBlobArea: 2000 +blobXgrowAmount: 20 +blobYgrowAmount: 10 +minimumGapBewtweenAdjacentBlobs: 50 +laserState: True +runningCamera: True +pump1State: False +pump2State: True +stirrer: False +viewImages: True +viewRoisOnly: True +sendTriggerContent: False +triggerContinuous: False +continuousTriggerRate: 1 +fill3newSheathState: False +PMTAtriggerThreshold_DAQ_MCConly: 0.15 +PMTBtriggerThreshold_DAQ_MCConly: 0.14 +PMTCtriggerThreshold_DAQ_MCConly: 0 +PMTDtriggerThreshold_DAQ_MCConly: 0 +ADCdataRateDAQ_MCConly: 240 +integratorSettleTime: 8 +PMTtriggerSelection_DAQ_MCConly: 3 +altPMTtriggerSelection_DAQ_MCConly: 3 +ADCPGA_DAQ_MCConly: 1 +triggerAckTimeout_x34720ns_DAQ_MCConly: 21600 +TimeoutForFirstFrame: 30 +pulseStretchTime_x2170ns_DAQ_MCConly: 21 +maximumPulseLength_x2170ns_DAQ_MCConly: 13823 +hasCamera: True +liveEnvironment: False +flash: True +PMTAhighVoltage: 0.65 +PMTBhighVoltage: 0.55 +intervalBetweenAlternateHardwareSettings: 0 +Alt_FlashlampControlVoltage: 3 +pumpDriveVoltage: 14 +altPMTAHighVoltage: 0.55 +altPMTBHighVoltage: 0.65 +altPMTATriggerThreshold_DAQ_MCConly: 0.13 +altPMTBTriggerThreshold_DAQ_MCConly: 0.13 +viewTriggers: True +viewGraphs: False +viewActivity: True +viewFPS: True +KloehnPort: /dev/ttyUSB0 +syringeSamplingSpeed: 20 +syringeOffset: 4176 +NumberSyringesToAutoRun: 3 +SyringeSampleVolume: 5 +altSyringeSampleVolume: 5 +sampleVolume2skip: 0 +syringeSize: 5 +primeVolume: 5 +flushVolume: 1 +BleachVolume: 1 +BiocideVolume: 1 +stepsToFullSyringe: 318354 +debubbleWithSample: True +RefillAfterDebubble: True +primeSampleTube: False +backflushWithSample: False +runBeads: False +runSampleFast: False +BeadsSampleVolume: 0.4 +NumberSyringesBetweenBeadsRun: 0 +NumberSyringesBetweenCleaningRun: 0 +RunFastFactor: 5 +viewSyringe: True +viewValve: True +focusMotorSmallStep_ms: 50 +focusMotorLargeStep_ms: 500 +laserMotorSmallStep_ms: 500 +laserMotorLargeStep_ms: 5000 +debubbleConeVolume: 2 +debubbleNeedleVolume: 3 +debubbleRefillVolume: 3.5 +debubbleLeftVolume: 0.5 +BleachRinseCount: 3 +BleachRinseVolume: 1 +BleachToExhaust: False +SamplingFastSpeed: 4 +CounterCleaning: 112 +CounterBeads: 112 +CounterAlt: 112 +imagerID: 174 +DataFilePath: /home/ifcb/ifcbdata/shimada_may2023 +FolderHierarchy: False +OutputFiles: True +FileComment: fsw +triggerThreadCount: 3 +featureThreadCount: 3 +FeaturesVersion: V4 +featureGeneration: False +SaveBlobs: False +WebBrowser: chromium +WebBrowserArguments: --disable-gpu +GPSFeed: 0 +PhytoArmDataSource: 0 +ServerAsAdmin: False +HostAsAdmin: False +UpdateServer: http://mclanelabs.dyndns.org:8082/external/update/ifcbacquire/release/ +TestMode: False diff --git a/ifcb/tests/test_files/D20231021T154635_IFCB199.hdr b/ifcb/tests/test_files/D20231021T154635_IFCB199.hdr new file mode 100644 index 0000000..2942dcb --- /dev/null +++ b/ifcb/tests/test_files/D20231021T154635_IFCB199.hdr @@ -0,0 +1,132 @@ +SoftwareVersion: 2.5.0.0 +AnalogFirmware: 45 +HousekeepingFirmware: 33 +sampleNumber: 2 +sampleType: Normal +triggerCount: 2 +roiCount: 0 +humidity: 40.3111390859846 +temperature: 22.28220721751736 +runTime: 240.95753472222222 +inhibitTime: 0 +ADCFileFormat: trigger#, ADCtime, PMTA, PMTB, PMTC, PMTD, PeakA, PeakB, PeakC, PeakD, TimeOfFlight, GrabTimeStart, GrabTimeEnd, RoiX, RoiY, RoiWidth, RoiHeight, StartByte, ComparatorOut, StartPoint, SignalLength, Status, RunTime, InhibitTime +DAQ_MCCserialPort_DAC_MCConly: /dev/ttyS0 +UvModulePresent: False +UvWithSample: False +auxPower1: False +autoStart: False +InteractiveAutoStart: False +autoShutdown: False +HumidityAlarmThreshold(%): 90 +ValveHumidityHysteresis_DAC_MCConly: 567 +FanTemperatureThreshold_DAC_MCConly: 30744 +FanTemperatureHysteresis_DAC_MCConly: 123 +valveDelay: 781 +FlashlampControlVoltage: 2.7999 +HKTRIGGERtoFlashlampDelayTime_x434ns_DAC_MCConly: 120 +FlashlampPulseLength_x434ns_DAC_MCConly: 20 +CameraPulseLength_x434ns_DAC_MCConly: 500 +cameraGain: 6 +binarizeThreshold: 6 +minimumBlobArea: 2000 +blobXgrowAmount: 20 +blobYgrowAmount: 10 +minimumGapBewtweenAdjacentBlobs: 50 +laserState: False +runningCamera: True +pump1State: True +pump2State: False +stirrer: False +viewImages: True +viewRoisOnly: False +sendTriggerContent: False +triggerContinuous: False +continuousTriggerRate: 3 +fill3newSheathState: False +PMTAtriggerThreshold_DAQ_MCConly: 0.15 +PMTBtriggerThreshold_DAQ_MCConly: 0.15 +PMTCtriggerThreshold_DAQ_MCConly: 0 +PMTDtriggerThreshold_DAQ_MCConly: 0 +ADCdataRateDAQ_MCConly: 240 +integratorSettleTime: 8 +PMTtriggerSelection_DAQ_MCConly: 2 +altPMTtriggerSelection_DAQ_MCConly: 2 +ADCPGA_DAQ_MCConly: 1 +triggerAckTimeout_x34720ns_DAQ_MCConly: 21600 +TimeoutForFirstFrame: 30 +pulseStretchTime_x2170ns_DAQ_MCConly: 21 +maximumPulseLength_x2170ns_DAQ_MCConly: 13823 +hasCamera: True +liveEnvironment: False +flash: True +PMTAhighVoltage: 0.45 +PMTBhighVoltage: 0.5 +intervalBetweenAlternateHardwareSettings: 0 +Alt_FlashlampControlVoltage: 3 +pumpDriveVoltage: 14 +altPMTAHighVoltage: 0.55 +altPMTBHighVoltage: 0.65 +altPMTATriggerThreshold_DAQ_MCConly: 0.13 +altPMTBTriggerThreshold_DAQ_MCConly: 0.13 +viewTriggers: True +viewGraphs: True +viewActivity: True +viewFPS: True +KloehnPort: /dev/ttyUSB0 +syringeSamplingSpeed: 20 +syringeOffset: 2765 +NumberSyringesToAutoRun: 5 +SyringeSampleVolume: 5 +altSyringeSampleVolume: 5 +sampleVolume2skip: 0 +syringeSize: 5 +primeVolume: 1 +flushVolume: 1 +BleachVolume: 1 +BiocideVolume: 1 +stepsToFullSyringe: 318354 +debubbleWithSample: True +RefillAfterDebubble: True +primeSampleTube: False +backflushWithSample: False +runBeads: False +runSampleFast: True +BeadsSampleVolume: 0.4 +NumberSyringesBetweenBeadsRun: 0 +NumberSyringesBetweenCleaningRun: 0 +RunFastFactor: 5 +viewSyringe: True +viewValve: True +focusMotorSmallStep_ms: 50 +focusMotorLargeStep_ms: 500 +laserMotorSmallStep_ms: 500 +laserMotorLargeStep_ms: 5000 +debubbleConeVolume: 2 +debubbleNeedleVolume: 3 +debubbleRefillVolume: 3.5 +debubbleLeftVolume: 0.5 +BleachRinseCount: 3 +BleachRinseVolume: 1 +BleachToExhaust: False +SamplingFastSpeed: 4 +CounterCleaning: 2 +CounterBeads: 2 +CounterAlt: 2 +imagerID: 199 +DataFilePath: /home/ifcb/ifcbdata +FolderHierarchy: False +OutputFiles: True +FileComment: file comment +triggerThreadCount: 3 +featureThreadCount: 3 +FeaturesVersion: V4 +featureGeneration: False +SaveBlobs: False +WebBrowser: chromium +WebBrowserArguments: --disable-gpu +GPSFeed: 0 +PhytoArmDataSource: 0 +ServerAsAdmin: False +HostAsAdmin: False +UpdateServer: http://mclanelabs.dyndns.org:8082/external/update/ifcbacquire/release/ +TestMode: False From e95e1ed2216664966af909a6b67c3a3470cfb907 Mon Sep 17 00:00:00 2001 From: Ian Black Date: Wed, 18 Jun 2025 09:47:17 -0700 Subject: [PATCH 3/3] Updates .hdr module with hints and new functions. - Updates file to conform more with PEP8. - Adds typing hints. - Adds non-public functions for seeking software version and parsing an hdr file as a yaml. --- ifcb/data/hdr.py | 66 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/ifcb/data/hdr.py b/ifcb/data/hdr.py index 6553fb3..7fb2676 100644 --- a/ifcb/data/hdr.py +++ b/ifcb/data/hdr.py @@ -1,13 +1,13 @@ """ Support for parsing IFCB header files. """ - -import re -import fileinput - import ast +import fileinput +from os import PathLike +import re +import yaml -HDR='hdr' +HDR = 'hdr' # hdr attributes. these are camel-case, mapped to column names below TEMPERATURE = 'temperature' @@ -15,7 +15,7 @@ BINARIZE_THRESHOLD = 'binarizeThreshold' SCATTERING_PMT_SETTING = 'scatteringPhotomultiplierSetting' FLUORESCENCE_PMT_SETTING = 'fluorescencePhotomultiplierSetting' -BLOB_SIZE_THRESHOLD = 'blobSizeThreshold' +BLOB_SIZE_THRESHOLD = 'blobSizeThreshold' # column name / type pairs HDR_SCHEMA = [(TEMPERATURE, float), @@ -29,7 +29,8 @@ CONTEXT = 'context' -def parse_alt_header(lines): + +def parse_alt_header(lines: list) -> dict: props = {} for line in lines: m = re.match(r'^run time = ([\d.]+) s\s+inhibit time = ([\d.]+)', line) @@ -43,7 +44,8 @@ def parse_alt_header(lines): props['humidity'] = float(m.group(2)) return props -def parse_hdr(lines): + +def parse_hdr(lines: list) -> dict: """ Given the lines of a header file, return the properties in it. @@ -56,12 +58,12 @@ def parse_hdr(lines): if lines[0] == 'Imaging FlowCytobot Acquisition Software version 2.0; May 2010': if lines[1].startswith('Sample Date'): return parse_alt_header(lines) - props = { CONTEXT: lines[0] } # FIXME parse - elif re.match(r'^[Ss]oftwareVersion:',lines[0]): - props = { CONTEXT: lines[0] } + props = {CONTEXT: lines[0]} # FIXME parse + elif re.match(r'^[Ss]oftwareVersion:', lines[0]): + props = {CONTEXT: lines[0]} for line in lines[1:]: try: - k, v = re.split(r': ',line) + k, v = re.split(r': ', line) try: v = ast.literal_eval(v) except ValueError: @@ -74,11 +76,11 @@ def parse_hdr(lines): pass else: # "context" is what the text on lines 2-4 is called in the header file - props = { CONTEXT: '\n'.join([line.strip('"') for line in lines[:-2]]) } + props = {CONTEXT: '\n'.join([line.strip('"') for line in lines[:-2]])} # now handle format variants - if len(lines) >= 6: # don't fail on original header format - columns = re.split(' +',re.sub('"','',lines[-2])) # columns of metadata in CSV format - values = re.split(' +',re.sub(r'[",]',' ',lines[-1]).strip()) # values of those columns in CSV format + if len(lines) >= 6: # don't fail on original header format + columns = re.split(' +', re.sub('"', '', lines[-2])) # columns of metadata in CSV format + values = re.split(' +', re.sub(r'[",]', ' ', lines[-1]).strip()) # values of those columns in CSV format # for each column take the string and cast it to the schema's column type for (column, (name, _), value) in zip(HDR_COLUMNS, HDR_SCHEMA, values): props[name] = value @@ -88,7 +90,37 @@ def parse_hdr(lines): props[name] = cast(props[name]) return props -def parse_hdr_file(path): + +def _get_software_version(hdr_filepath: PathLike) -> str: + """ + Return the software version that is within an IFCB .hdr file. + + :param hdr_filepath: A path-like string to the .hdr file. + :return: The software version as a string. Typically, in the format of N.N.N.N representing major.minor.patch.build. + """ + with open(hdr_filepath, 'r') as _file: + lines = _file.readlines() + for line in lines: + if 'software' in line.lower(): + version = re.findall(r'([0-9.]+)', line)[0] + return version + raise IOError(f"Software version information not found in header: {hdr_filepath}") + + +def _parse_hdr(hdr_filepath: PathLike) -> dict: + """ + Parse a .hdr file as YAML. + + :param hdr_filepath: A path-like string to the .hdr file. + :returns dict: A dictionary containing the parsed header data. + """ + + with open(hdr_filepath, 'r') as _file: + hdr_data = yaml.safe_load(_file) + return hdr_data + + +def parse_hdr_file(path: PathLike) -> dict: """ Given a path to a header file, return the header properties.