feat(native): offline caching

CHANGELOG.md

@@ -5,9 +5,11 @@
 **Features**:
 
 - Symbolicate stack frames in crash daemon on Windows. ([#1595](https://github.com/getsentry/sentry-native/pull/1595))
+- Add offline caching support to the new experimental `native` backend. ([#1585](https://github.com/getsentry/sentry-native/pull/1585))
 
 **Fixes**:
 
+- Fix `cache_keep` to only cache envelopes when HTTP send fails, instead of unconditionally on restart. ([#1585](https://github.com/getsentry/sentry-native/pull/1585))
 - Fix external crash reporter to work with the new experimental `native` backend. ([#1589](https://github.com/getsentry/sentry-native/pull/1589))
 
 ## 0.13.3

examples/example.c

@@ -940,6 +940,10 @@ main(int argc, char **argv)
         sentry_reinstall_backend();
     }
 
+    if (has_arg(argc, argv, "flush")) {
+        sentry_flush(10000);
+    }
+
     if (has_arg(argc, argv, "sleep")) {
         sleep_s(10);
     }

include/sentry.h

@@ -1476,12 +1476,14 @@ SENTRY_API int sentry_options_get_symbolize_stacktraces(
     const sentry_options_t *opts);
 
 /**
- * Enables or disables storing envelopes in a persistent cache.
+ * Enables or disables storing envelopes that fail to send in a persistent
+ * cache.
  *
- * When enabled, envelopes are written to a `cache/` subdirectory within the
- * database directory and retained regardless of send success or failure.
- * The cache is cleared on startup based on the cache_max_items, cache_max_size,
- * and cache_max_age options.
+ * When enabled, envelopes that fail to send are written to a `cache/`
+ * subdirectory within the database directory. The cache is cleared on startup
+ * based on the cache_max_items, cache_max_size, and cache_max_age options.
+ *
+ * Only applicable for HTTP transports.
  *
  * Disabled by default.
  */

src/backends/native/sentry_crash_context.h

@@ -260,6 +260,7 @@ typedef struct {
     int crash_reporting_mode; // sentry_crash_reporting_mode_t
     bool debug_enabled; // Debug logging enabled in parent process
     bool attach_screenshot; // Screenshot attachment enabled in parent process
+    bool cache_keep;
 
     // Platform-specific crash context
 #if defined(SENTRY_PLATFORM_LINUX) || defined(SENTRY_PLATFORM_ANDROID)

src/backends/native/sentry_crash_daemon.c

@@ -3040,6 +3040,7 @@ sentry__crash_daemon_main(pid_t app_pid, uint64_t app_tid, HANDLE event_handle,
     // Use debug logging and screenshot settings from parent process
     sentry_options_set_debug(options, ipc->shmem->debug_enabled);
     options->attach_screenshot = ipc->shmem->attach_screenshot;
+    options->cache_keep = ipc->shmem->cache_keep;
 
     // Set custom logger that writes to file
     if (log_file) {

src/backends/sentry_backend_crashpad.cpp

@@ -565,11 +565,9 @@ process_completed_reports(
 
     SENTRY_DEBUGF("caching %zu completed reports", reports.size());
 
-    sentry_path_t *cache_dir
-        = sentry__path_join_str(options->database_path, "cache");
-    if (!cache_dir || sentry__path_create_dir_all(cache_dir) != 0) {
+    sentry_path_t *cache_dir = options->run->cache_path;
+    if (sentry__path_create_dir_all(cache_dir) != 0) {
         SENTRY_WARN("failed to create cache dir");
-        sentry__path_free(cache_dir);
         return;
     }
 
@@ -593,8 +591,6 @@ process_completed_reports(
         sentry__path_free(out_path);
         sentry_envelope_free(envelope);
     }
-
-    sentry__path_free(cache_dir);
 }
 
 static int

src/backends/sentry_backend_native.c

@@ -176,6 +176,7 @@ native_backend_startup(
     // Pass debug logging setting to daemon
     ctx->debug_enabled = options->debug;
     ctx->attach_screenshot = options->attach_screenshot;
+    ctx->cache_keep = options->cache_keep;
 
     // Set up event and breadcrumb paths
     sentry_path_t *run_path = options->run->run_path;

src/sentry_core.c

@@ -293,7 +293,9 @@ sentry_init(sentry_options_t *options)
     }
 
     if (options->cache_keep) {
-        sentry__cleanup_cache(options);
+        if (!sentry__transport_submit_cleanup(options->transport, options)) {
+            sentry__cleanup_cache(options);
+        }
     }
 
     if (options->auto_session_tracking) {

src/sentry_database.c

@@ -4,6 +4,8 @@
 #include "sentry_json.h"
 #include "sentry_options.h"
 #include "sentry_session.h"
+#include "sentry_sync.h"
+#include "sentry_utils.h"
 #include "sentry_uuid.h"
 #include <errno.h>
 #include <stdlib.h>
@@ -50,19 +52,32 @@ sentry__run_new(const sentry_path_t *database_path)
         return NULL;
     }
 
+    // `<db>/cache`
+    sentry_path_t *cache_path = sentry__path_join_str(database_path, "cache");
+    if (!cache_path) {
+        sentry__path_free(run_path);
+        sentry__path_free(lock_path);
+        sentry__path_free(session_path);
+        sentry__path_free(external_path);
+        return NULL;
+    }
+
     sentry_run_t *run = SENTRY_MAKE(sentry_run_t);
     if (!run) {
         sentry__path_free(run_path);
         sentry__path_free(session_path);
         sentry__path_free(lock_path);
         sentry__path_free(external_path);
+        sentry__path_free(cache_path);
         return NULL;
     }
 
+    run->refcount = 1;
     run->uuid = uuid;
     run->run_path = run_path;
     run->session_path = session_path;
     run->external_path = external_path;
+    run->cache_path = cache_path;
     run->lock = sentry__filelock_new(lock_path);
     if (!run->lock) {
         goto error;
@@ -80,6 +95,15 @@ sentry__run_new(const sentry_path_t *database_path)
     return NULL;
 }
 
+sentry_run_t *
+sentry__run_incref(sentry_run_t *run)
+{
+    if (run) {
+        sentry__atomic_fetch_and_add(&run->refcount, 1);
+    }
+    return run;
+}
+
 void
 sentry__run_clean(sentry_run_t *run)
 {
@@ -90,12 +114,13 @@ sentry__run_clean(sentry_run_t *run)
 void
 sentry__run_free(sentry_run_t *run)
 {
-    if (!run) {
+    if (!run || sentry__atomic_fetch_and_add(&run->refcount, -1) != 1) {
         return;
     }
     sentry__path_free(run->run_path);
     sentry__path_free(run->session_path);
     sentry__path_free(run->external_path);
+    sentry__path_free(run->cache_path);
     sentry__filelock_free(run->lock);
     sentry_free(run);
 }
@@ -151,6 +176,18 @@ sentry__run_write_external(
     return write_envelope(run->external_path, envelope);
 }
 
+bool
+sentry__run_write_cache(
+    const sentry_run_t *run, const sentry_envelope_t *envelope)
+{
+    if (sentry__path_create_dir_all(run->cache_path) != 0) {
+        SENTRY_ERRORF("mkdir failed: \"%s\"", run->cache_path->path);
+        return false;
+    }
+
+    return write_envelope(run->cache_path, envelope);
+}
+
 bool
 sentry__run_write_session(
     const sentry_run_t *run, const sentry_session_t *session)
@@ -239,14 +276,6 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash)
             continue;
         }
 
-        sentry_path_t *cache_dir = NULL;
-        if (options->cache_keep) {
-            cache_dir = sentry__path_join_str(options->database_path, "cache");
-            if (cache_dir) {
-                sentry__path_create_dir_all(cache_dir);
-            }
-        }
-
         sentry_pathiter_t *run_iter = sentry__path_iter_directory(run_dir);
         const sentry_path_t *file;
         while (run_iter && (file = sentry__pathiter_next(run_iter)) != NULL) {
@@ -291,25 +320,12 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash)
             } else if (sentry__path_ends_with(file, ".envelope")) {
                 sentry_envelope_t *envelope = sentry__envelope_from_path(file);
                 sentry__capture_envelope(options->transport, envelope);
-
-                if (cache_dir) {
-                    sentry_path_t *cached_file = sentry__path_join_str(
-                        cache_dir, sentry__path_filename(file));
-                    if (!cached_file
-                        || sentry__path_rename(file, cached_file) != 0) {
-                        SENTRY_WARNF("failed to cache envelope \"%s\"",
-                            sentry__path_filename(file));
-                    }
-                    sentry__path_free(cached_file);
-                    continue;
-                }
             }
 
             sentry__path_remove(file);
         }
         sentry__pathiter_free(run_iter);
 
-        sentry__path_free(cache_dir);
         sentry__path_remove_all(run_dir);
         sentry__filelock_free(lock);
     }

src/sentry_database.h

@@ -11,7 +11,9 @@ typedef struct sentry_run_s {
     sentry_path_t *run_path;
     sentry_path_t *session_path;
     sentry_path_t *external_path;
+    sentry_path_t *cache_path;
     sentry_filelock_t *lock;
+    long refcount;
 } sentry_run_t;
 
 /**
@@ -22,6 +24,11 @@ typedef struct sentry_run_s {
  */
 sentry_run_t *sentry__run_new(const sentry_path_t *database_path);
 
+/**
+ * Increment the refcount and return the run pointer.
+ */
+sentry_run_t *sentry__run_incref(sentry_run_t *run);
+
 /**
  * This will clean up all the files belonging to this run.
  */
@@ -63,6 +70,13 @@ bool sentry__run_write_session(
  */
 bool sentry__run_clear_session(const sentry_run_t *run);
 
+/**
+ * This will serialize and write the given envelope to disk into the cache
+ * directory: `<database>/cache/<event-uuid>.envelope`
+ */
+bool sentry__run_write_cache(
+    const sentry_run_t *run, const sentry_envelope_t *envelope);
+
 /**
  * This function is essential to send crash reports from previous runs of the
  * program.

src/sentry_sync.c

@@ -170,6 +170,7 @@ struct sentry_bgworker_s {
     void (*free_state)(void *state);
     long refcount;
     long running;
+    long draining;
 };
 
 sentry_bgworker_t *
@@ -236,6 +237,9 @@ sentry__bgworker_get_state(sentry_bgworker_t *bgw)
 static bool
 sentry__bgworker_is_done(sentry_bgworker_t *bgw)
 {
+    if (sentry__atomic_fetch(&bgw->draining)) {
+        return true;
+    }
     return (!bgw->first_task
                || sentry__monotonic_time() < bgw->first_task->execute_after)
         && !sentry__atomic_fetch(&bgw->running);
@@ -318,6 +322,7 @@ sentry__bgworker_start(sentry_bgworker_t *bgw)
 {
     SENTRY_DEBUG("starting background worker thread");
     sentry__atomic_store(&bgw->running, 1);
+    sentry__atomic_store(&bgw->draining, 0);
     // this incref moves the reference into the background thread
     sentry__bgworker_incref(bgw);
     if (sentry__thread_spawn(&bgw->thread_id, &worker_thread, bgw) != 0) {
@@ -425,7 +430,8 @@ shutdown_task(void *task_data, void *UNUSED(state))
 }
 
 int
-sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout)
+sentry__bgworker_shutdown_cb(sentry_bgworker_t *bgw, uint64_t timeout,
+    void (*on_timeout)(void *), void *on_timeout_data)
 {
     if (!sentry__atomic_fetch(&bgw->running)) {
         SENTRY_WARN("trying to shut down non-running thread");
@@ -441,12 +447,24 @@ sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout)
     while (true) {
         uint64_t now = sentry__monotonic_time();
         if (now > started && now - started > timeout) {
-            sentry__atomic_store(&bgw->running, 0);
-            sentry__thread_detach(bgw->thread_id);
-            sentry__mutex_unlock(&bgw->task_lock);
-            SENTRY_WARN(
-                "background thread failed to shut down cleanly within timeout");
-            return 1;
+            if (on_timeout) {
+                // fire on_timeout to cancel the ongoing task, and give the
+                // worker an extra loop cycle up to 250ms to handle the
+                // cancellation
+                sentry__mutex_unlock(&bgw->task_lock);
+                on_timeout(on_timeout_data);
+                sentry__atomic_store(&bgw->draining, 1);
+                on_timeout = NULL;
+                sentry__mutex_lock(&bgw->task_lock);
+                // fall through to !running check below
+            } else {
+                sentry__atomic_store(&bgw->running, 0);
+                sentry__thread_detach(bgw->thread_id);
+                sentry__mutex_unlock(&bgw->task_lock);
+                SENTRY_WARN("background thread failed to shut down cleanly "
+                            "within timeout");
+                return 1;
+            }
         }
 
         if (!sentry__atomic_fetch(&bgw->running)) {
@@ -553,7 +571,7 @@ sentry__bgworker_foreach_matching(sentry_bgworker_t *bgw,
         bool drop_task = false;
         // only consider tasks matching this exec_func
         if (task->exec_func == exec_func) {
-            drop_task = callback(task->task_data, data);
+            drop_task = !callback || callback(task->task_data, data);
         }
 
         sentry_bgworker_task_t *next_task = task->next_task;

src/sentry_sync.h

@@ -469,8 +469,20 @@ int sentry__bgworker_flush(sentry_bgworker_t *bgw, uint64_t timeout);
 /**
  * This will try to shut down the background worker thread, with a `timeout`.
  * Returns 0 on success.
+ *
+ * The `_cb` variant accepts an `on_timeout` callback that is invoked when
+ * the timeout expires, just before detaching the thread. This gives the
+ * caller a chance to unblock the worker (e.g. by closing transport handles)
+ * so it can finish in-flight work.
  */
-int sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout);
+int sentry__bgworker_shutdown_cb(sentry_bgworker_t *bgw, uint64_t timeout,
+    void (*on_timeout)(void *), void *on_timeout_data);
+
+static inline int
+sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout)
+{
+    return sentry__bgworker_shutdown_cb(bgw, timeout, NULL, NULL);
+}
 
 /**
  * This will set a preferable thread name for background worker.