Raise errors in Otel/Sentry

config/runtime.exs

@@ -626,7 +626,7 @@ ch_transport_opts =
 config :plausible, Plausible.ClickhouseRepo,
   queue_target: 500,
   queue_interval: 2000,
-  timeout: 15_000,
+  timeout: 1_000,
   url: ch_db_url,
   transport_opts: ch_transport_opts,
   settings: [
@@ -638,7 +638,7 @@ config :plausible, Plausible.ClickhouseRepo,
     # NB! when :timeout is overridden to be over 20s,
     # for it to have meaningful effect,
     # this must be overridden as well
-    max_execution_time: 20
+    max_execution_time: 2
   ]
 
 config :plausible, Plausible.IngestRepo,

lib/plausible/stats/query_runner.ex

@@ -12,6 +12,9 @@ defmodule Plausible.Stats.QueryRunner do
   use Plausible
   use Plausible.ClickhouseRepo
 
+  require Logger
+  require OpenTelemetry.Tracer, as: Tracer
+
   alias Plausible.Stats.{
     Comparisons,
     Compare,
@@ -111,9 +114,37 @@ defmodule Plausible.Stats.QueryRunner do
   end
 
   defp execute_query(query, site) do
-    query
-    |> SQL.QueryBuilder.build(site)
-    |> ClickhouseRepo.all(query: query)
+    try do
+      query
+      |> SQL.QueryBuilder.build(site)
+      |> ClickhouseRepo.all(query: query)
+    rescue
+      e in [Ch.Error, DBConnection.ConnectionError, Mint.TransportError] ->
+        message = Exception.message(e)
+        Logger.error("ClickHouse query error: #{message}")
+
+        Tracer.set_status(:error, message)
+
+        Tracer.set_attributes([
+          {"plausible.query.error", message},
+          {"plausible.query.error.kind", inspect(e.__struct__)}
+        ])
+
+        Sentry.Context.set_extra_context(%{
+          site_domain: site.domain,
+          site_id: site.id,
+          metrics: inspect(query.metrics),
+          dimensions: inspect(query.dimensions),
+          date_range: inspect(query.utc_time_range),
+          filters: inspect(query.filters)
+        })
+
+        Sentry.Context.set_tags_context(%{
+          source: "clickhouse_query_timeout"
+        })
+
+        reraise e, __STACKTRACE__
+    end
   end
 
   defp build_from_ch(ch_results, query) do

lib/plausible_web/plugs/error_handler.ex

@@ -12,9 +12,13 @@ defmodule PlausibleWeb.Plugs.ErrorHandler do
       use Plug.ErrorHandler
 
       @impl Plug.ErrorHandler
-      def handle_errors(conn, %{kind: kind, reason: reason}) do
+      def handle_errors(conn, %{kind: :error, reason: reason, stack: stack}) do
+        OpenTelemetry.Tracer.set_status(:error, Exception.message(reason))
+        Sentry.capture_exception(reason, stacktrace: stack, handled: false)
         json(conn, %{error: "internal server error"})
       end
+
+      def handle_errors(conn, _), do: json(conn, %{error: "internal server error"})
     end
   end
 end