pnsn · ulbergc · Jun 13, 2017 · Dec 21, 2022 · Dec 21, 2022 · Jan 3, 2023
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -6,7 +6,7 @@ ENABLE_LANGUAGE(CXX)
 ENABLE_TESTING()
 SET(GFAST_VERSION_MAJOR 1)
 SET(GFAST_VERSION_MINOR 2)
-SET(GFAST_VERSION_PATCH 3)
+SET(GFAST_VERSION_PATCH 4)
 SET(GFAST_VERSION ${GFAST_VERSION_MAJOR}.${GFAST_VERSION_MINOR}.${GFAST_VERSION_PATCH})
 IF ("${GFAST_INSTANCE}" STREQUAL "")
   SET(GFAST_INSTANCE "DEFAULT")
@@ -114,7 +114,8 @@ SET(SRCS_CORE
 #ADD_SUBDIRECTORY(src/eewUtils)
 SET(SRCS_EEW src/eewUtils/driveCMT.c src/eewUtils/driveFF.c src/eewUtils/driveGFAST.c
              src/eewUtils/drivePGD.c src/eewUtils/makeXML.c src/eewUtils/parseCoreXML.c
-             src/eewUtils/setLogFileNames.c)
+             src/eewUtils/setLogFileNames.c src/eewUtils/fillCoreEventInfo.c
+             src/eewUtils/sendXMLFilter.c)
 #ADD_SUBDIRECTORY(src/traceBuffer)
 SET(SRCS_H5TB src/traceBuffer/h5/copyTraceBufferToGFAST.c src/traceBuffer/h5/finalize.c
               src/traceBuffer/h5/getData.c src/traceBuffer/h5/getDoubleArray.c

diff --git a/CMakeModules/FindXML2.cmake b/CMakeModules/FindXML2.cmake
@@ -19,7 +19,7 @@ find_library(LIBXML2_LIBRARY NAMES xml2 libxml2
 include(FindPackageHandleStandardArgs)
 # handle the QUIETLY and REQUIRED arguments and set LIBXML2_FOUND to TRUE
 # if all listed variables are TRUE
-find_package_handle_standard_args(LibXml2  DEFAULT_MSG
+find_package_handle_standard_args(XML2  DEFAULT_MSG
                                   LIBXML2_LIBRARY LIBXML2_INCLUDE_DIR)
 
 mark_as_advanced(LIBXML2_INCLUDE_DIR LIBXML2_LIBRARY )

diff --git a/ChangeLog b/ChangeLog
@@ -0,0 +1,83 @@
+version 1.2.4 2023-08-03  Carl Ulberg   <ulbergc@uw.edu>
+    * Summary:
+      - Add internal/external message versions in driveGFAST. The internal versions start at v-1,
+        then decrement by 1 on every single iteration. The external version numbers start at v0,
+        and increment by 1 on pgd xml messages that are actually sent to the Solution Aggregator.
+        Every message is still saved in the run/out directory - if the message was sent to the SA,
+        it will have the positive, external version number. If it was not sent, it will have the
+        negative, internal version number.
+      - Change how pgd xml timestamp is defined. Previously it represented the time at the start
+        of the current iteration. Now it represents the time closest to when the message is
+        actually sent (really from when the message is generated, but then it is sent in the next
+        step).
+      - Add publish change thresholds. Among the other send filter logic, the change thresholds
+        ensure that GFAST won't send identical (or nearly identical) messages to the SA.
+      - Refactor some code out of driveGFAST (easier to unit test), fillCoreEventInfo and
+        sendXMLFilter
+      - New unit tests
+version 1.2.3 2023-04-10  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Fully use Q channel with "goodness/use" flag. If configuration is set, GFAST will ignore
+        data points and PGD observations with a use flag of 0, as determined by the CWU/Fastlane
+        processing.
+      - Use "sleep" instead of "while true" loop for each 1s iteration. Saves processor time,
+        although may lead to slight creep in iteration start time.
+version 1.2.2 2022-12-28  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Bring Q channel through processing. Not entirely set up, but the Q channel is mostly
+        available when calculating PGD, along with the 6 other channels: ENZ/ENZ uncertainty.
+version 1.2.1 2022-11-28  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Add hashmap to tb2 processing to speed up get/unpack messages.
+      - Add raw observation uncertainty thresholds in PGD processing. This is intended to handle
+        clock correction steps in the data.
+version 1.2.0 2022-10-04  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Add GFAST to ShakeAlert repo
+      - Use DMMessageReceiver for reading AMQ messages.
+      - Add GFAST to ShakeAlert unit test coverage report. 
+      - Add config options for turning on/off pgd/cmt/ff processing. 
+      - Reduce hdf5 archive size
+### Versions before this were in the PNSN github repo at https://github.com/pnsn/GFAST ###
+version 1.1.10 2022-07-27  Carl Ulberg   <ulbergc@uw.edu>
+    * Summary:
+      - logging updates, etc before adding to ShakeAlert repo
+version 1.1.9 2022-06-29  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - pgd_mag_sigma throttling and updated instance header
+version 1.1.8 2022-05-27  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - add 'assoc' attribute to pgd xml
+version 1.1.7 2022-04-28  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - min/max pgd thresholds
+version 1.1.6 2022-04-01  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - update cmt quakeml
+version 1.1.5 2022-03-01  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - add time/mag based magnitude uncertainty
+version 1.1.4b 2022-02-18  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - throttle based on time-varying pgd_obs
+version 1.1.4 2022-01-14  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - throttle based on pgd_obs
+version 1.1.3b 2021-12-22  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - use nan uncert
+version 1.1.3 2021-12-02  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Throttle xml based on SA mag
+version 1.1.2 2021-11-17  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Add strike, dip to ff xml message (using old xml writer)
+version 1.1.1 2021-11-09  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - bug fix from 1.1.0 to only add pgd observations that were used in pgd inversion
+version 1.1.0 2021  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Use dmlib to encode pgd xml, with pgd observations
+version 1.0.0 2021  Carl Ulberg    <ulbergc@uw.edu>
+    * Summary:
+      - Initial merge of SAdev and 2020 branches from GFAST PNSN github repo
diff --git a/README.md b/README.md
@@ -4,6 +4,7 @@ This is the source code for Geodetic First Approximation of Size and Timing (GFA
 
 ## Recent updates
 
+2023/12/20 CWU: Sync most recent changes from ShakeAlert GFAST version to the PNSN repo (GFAST v1.2.4)
 2023/03/20 CWU: Sync most recent changes from ShakeAlert GFAST version to the PNSN repo (GFAST v1.2.3-beta)  
 ~2021/10 CWU: merge 2020 and SAdev branches into one branch to use going forward, called "development"  
 2020/05/27 MTH: Working on new branch (=2020) with cleaned up dependencies and build instructions.
@@ -14,14 +15,15 @@ This is the source code for Geodetic First Approximation of Size and Timing (GFA
 2. include contains the GFAST C include files.
 3. legacy is the original Python source code.
 4. src contains the source code.
-  + src/activeMQ contains C++ readers/writers and C interfaes for using ActiveMQ.
+  + src/activeMQ contains C++ readers/writers and C interfaces for using ActiveMQ.
   + src/core contains the core GFAST computations.
   + src/dmlib contains functionality making use of some ShakeAlert libraries, primarily related to ActiveMQ connections and xml encoding/decoding. Access to the ShakeAlert code repository would be necessary to build using these codes.
   + src/eewUtils contains application specific functions for performing the earthquake early warning tasks.
   + src/hdf5 contains the HDF5 interfaces for generating a self-describing archive or play-by-play of GFAST.
   + src/traceBuffer contains routines for reading an Earthworm ring and converting to a GFAST specific buffer.
   + src/uw contains functions specific to the University of Washington and Amazon project.
   + src/xml contains functions for certainly writing and potentially reading QuakeML and ShakeAlert specific XML.
+  + src/tests contains googletest unit tests, primarily tuned to work within a ShakeAlert environment/repo
 5. unit\_tests contains some simple regression tests for the core modules.
 
 # Building GFAST 
@@ -186,7 +188,7 @@ But here's how to do it in stand-alone
         >make              // will build lib/libiscl_shared.so and lib/libiscl_static.a
 
 
-7. [compearth/mtbeach](https://github.com/carltape/mtbeach.git) for the moment tensor decompositions.  Note, this was in Ben Baker's [compearth repository](https://github.com/bakerb845/compearth), but was merged back into Carl Tape's repository, and the moment tensor part was renames mtbeach.  Also, you'll need to descend into c_src. 
+7. [compearth/mtbeach](https://github.com/carltape/mtbeach.git) for the moment tensor decompositions.  Note, this was in Ben Baker's [compearth repository](https://github.com/bakerb845/compearth), but was merged back into Carl Tape's repository, and the moment tensor part was renamed mtbeach.  Also, you'll need to descend into c_src. 
 
         Clone and build
         >git clone https://github.com/carltape/mtbeach.git
@@ -243,7 +245,7 @@ But here's how to do it in stand-alone
 
     9a. [ActiveMQ](http://activemq.apache.org/) The C++ portion.  These will require other things that you likely already have like libssl, libcrypto, and the Apache runtime library.
 
-    Note: [MTH] I haven't found the APR to be necessary
+    Note: [MTH] I haven't found the APR to be necessary  
     Note: For building GFAST, the instructions here for downloading and
           installing activemq-cpp are sufficient.
           However, as soon as gfast_eew starts, it will try to connect

diff --git a/include/gfast_activeMQ.h b/include/gfast_activeMQ.h
@@ -50,10 +50,6 @@ void activeMQ_producer_finalize(void *producerIn);
 /* Send a text message */
 int activeMQ_producer_sendMessage(void *producerIn,
                                   const char *message);
-/* Convenience function to set the tcp URI request */
-char *activeMQ_setTcpURIRequest(const char *url,
-                                const int msReconnect,
-                                const int maxAttempts);
 /* Initialize the ActiveMQ producer */
 void *activeMQ_producer_initialize(const char AMQuser[],
                                    const char AMQpassword[],
@@ -63,6 +59,11 @@ void *activeMQ_producer_initialize(const char AMQuser[],
                                    const bool clientAck,
                                    const int verbose,
                                    int *ierr);
+/* Convenience function to set the tcp URI request */
+char *activeMQ_setTcpURIRequest(const char *url,
+                                const int msReconnect,
+                                const int maxAttempts);
+
 
 #ifndef __cplusplus
 #define GFAST_activeMQ_consumer_initialize(...)       \

diff --git a/include/gfast_config.h b/include/gfast_config.h
@@ -1,7 +1,7 @@
 #ifndef _gfast_config__h_
 #define _gfast_config__h_ 1
-#define GFAST_VERSION "gfast-1.2.3-beta"
-#define GFAST_MAXMSG_LEN 4096
+#define GFAST_VERSION "gfast-1.2.4-2023-10-12"
+#define GFAST_MAXMSG_LEN 8192
 #define MAX_OUTPUT_INTERVALS 16
 #define MAX_THROTTLING_THRESHOLDS 100
 #define MAX_SIGMA_LOOKUP_VALUES 100

diff --git a/include/gfast_core.h b/include/gfast_core.h
@@ -153,10 +153,10 @@ int core_data_readSiteMaskFile(const char *siteMaskFile,
 //----------------------------------------------------------------------------//
 /* Frees memory on an event structure */
 void core_events_freeEvents(struct GFAST_activeEvents_struct *events);
-/* Convenience function to find min origin time in event list */
-double core_events_getMinOriginTime(struct GFAST_props_struct props,
-                                    struct GFAST_activeEvents_struct events,
-                                    bool *lnoEvents);
+// /* Convenience function to find min origin time in event list */
+// double core_events_getMinOriginTime(struct GFAST_props_struct props,
+//                                     struct GFAST_activeEvents_struct events,
+//                                     bool *lnoEvents);
 /* Adds a new event to the event list */
 bool core_events_newEvent(struct GFAST_shakeAlert_struct SA, 
                           struct GFAST_activeEvents_struct *events,
@@ -166,12 +166,12 @@ bool core_events_syncXMLStatusWithEvents(struct GFAST_activeEvents_struct *event
 
 /* Print the events in the event list */
 void core_events_printEvents(struct GFAST_shakeAlert_struct SA);
-/* Remove a cancelled event from the events list */
-bool core_events_removeCancelledEvent(const char *evid,
-                                      const double currentTime,
-                                      const int verbose,
-                                      struct GFAST_shakeAlert_struct SA,
-                                      struct GFAST_activeEvents_struct *events);
+// /* Remove a cancelled event from the events list */
+// bool core_events_removeCancelledEvent(const char *evid,
+//                                       const double currentTime,
+//                                       const int verbose,
+//                                       struct GFAST_shakeAlert_struct SA,
+//                                       struct GFAST_activeEvents_struct *events);
 /* Look through the events list and remove expired events */
 int core_events_removeExpiredEvents(const double maxTime, 
                                     const double currenTime,
@@ -183,10 +183,10 @@ bool core_events_removeExpiredEvent(const double maxtime,
                                     const int verbose,
                                     struct GFAST_shakeAlert_struct SA,
                                     struct GFAST_activeEvents_struct *events);
-/* Potentially add the shakeAlert event to the event list */
-bool core_events_updateEvent(struct GFAST_shakeAlert_struct SA, 
-                             struct GFAST_activeEvents_struct *events,
-                             int *ierr);
+// /* Potentially add the shakeAlert event to the event list */
+// bool core_events_updateEvent(struct GFAST_shakeAlert_struct SA, 
+//                              struct GFAST_activeEvents_struct *events,
+//                              int *ierr);
 
 
 //----------------------------------------------------------------------------//
@@ -626,9 +626,12 @@ int core_scaling_readRawSigmaThresholdLookupFile(const char *rawSigmaThresholdLo
 /* Parse the packed quality channel, return chi^2 value */
 double core_waveformProcessor_parseQChannelChi2CWU(
     const double q_value);
-/* Parse the packed quality channel, return mapped chi^2 value*/
+/* Parse the packed quality channel, return mapped chi^2 value */
 int core_waveformProcessor_parseQChannelChi2CWUmap(
     const double q_value);
+/* Parse the packed quality channel, return goodness (0 or 1) */
+int core_waveformProcessor_parseQChannelGoodness(
+    const double q_value);
 /* Compute the offset in north, east, and up */
 int core_waveformProcessor_offset(const int utm_zone,
                                   const double svel_window,
@@ -705,24 +708,23 @@ double core_waveformProcessor_peakDisplacementHelper(
               core_data_readMetaDataFile(__VA_ARGS__)
 #define GFAST_core_data_readSiteMaskFile(...)       \
               core_data_readSiteMaskFile(__VA_ARGS__)
-
+              
 #define GFAST_core_events_freeEvents(...)       \
               core_events_freeEvents(__VA_ARGS__)
-#define GFAST_core_events_getMinOriginTime(...)       \
-              core_events_getMinOriginTime(__VA_ARGS__)
 #define GFAST_core_events_newEvent(...)       \
               core_events_newEvent(__VA_ARGS__)
 #define GFAST_core_events_syncXMLStatusWithEvents(...)       \
               core_events_syncXMLStatusWithEvents(__VA_ARGS__)
-
 #define GFAST_core_events_printEvents(...)       \
               core_events_printEvents(__VA_ARGS__)
-#define GFAST_core_events_removeCancelledEvent(...)       \
-              core_events_removeCancelledEvent(__VA_ARGS__)
 #define GFAST_core_events_removeExpiredEvent(...)       \
               core_events_removeExpiredEvent(__VA_ARGS__)
-#define GFAST_core_events_updateEvent(...)       \
-              core_events_updateEvent(__VA_ARGS__)
+// #define GFAST_core_events_getMinOriginTime(...)       
+//               core_events_getMinOriginTime(__VA_ARGS__)
+// #define GFAST_core_events_removeCancelledEvent(...)       
+//               core_events_removeCancelledEvent(__VA_ARGS__)
+// #define GFAST_core_events_updateEvent(...)       
+//               core_events_updateEvent(__VA_ARGS__)
 
 #define GFAST_core_ff_faultPlaneGridSearch(...)       \
               core_ff_faultPlaneGridSearch(__VA_ARGS__)

diff --git a/include/gfast_eewUtils.h b/include/gfast_eewUtils.h
@@ -96,6 +96,31 @@ void eewUtils_setLogFileNames(const char *eventid,
                               char infoLogFileName[PATH_MAX],
                               char debugLogFileName[PATH_MAX],
                               char warnLogFileName[PATH_MAX]);
+/* Fill a coreInfo struct with the given values */
+int eewUtils_fillCoreEventInfo(
+    const char *evid,
+    const int version,
+    const double SA_lat,
+    const double SA_lon,
+    const double SA_depth,
+    const double SA_mag,
+    const double SA_time,
+    const int num_stations,
+    struct coreInfo_struct *core);
+/* Helper function for eewUtils_sendXMLFilter, check change thresholds */
+bool eewUtils_changeThresholdsExceeded(
+    const struct GFAST_props_struct *props,
+    const struct coreInfo_struct *core,
+    const struct coreInfo_struct *last_sent_core);
+/* Determine if PGD message should be sent */
+bool eewUtils_sendXMLFilter(
+    const struct GFAST_props_struct *props,
+    const struct GFAST_shakeAlert_struct *SA,
+    const struct GFAST_pgdResults_struct *pgd,
+    const struct GFAST_peakDisplacementData_struct *pgd_data,
+    const struct coreInfo_struct *core,
+    const struct coreInfo_struct *last_sent_core,
+    const double age_of_event);
 
 #define GFAST_eewUtils_driveCMT(...)       \
               eewUtils_driveCMT(__VA_ARGS__)