diff --git a/CHANGELOG.md b/CHANGELOG.md index 3633eed60d3f..9ee6532a97a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- Allow querying `views_per_visit` with a time dimension in Stats API - Add `bounce_rate` to page-filtered Top Stats even when imports are included, but render a metric warning about imported data not included in `bounce_rate` tooltip. - Add `time_on_page` to page-filtered Top Stats even when imports are included, unless legacy time on page is in view. - Adds team_id to query debug metadata (saved in system.query_log log_comment column) @@ -20,6 +21,7 @@ All notable changes to this project will be documented in this file. ### Fixed +- Fixed Stats API timeseries returning time buckets falling outside the queried range - Fixed issue with all non-interactive events being counted as interactive - Fixed countries map countries staying highlighted on Chrome diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.ex index 1aea69dcb437..53af31245ce8 100644 --- a/lib/plausible/stats/api_query_parser.ex +++ b/lib/plausible/stats/api_query_parser.ex @@ -221,14 +221,14 @@ defmodule Plausible.Stats.ApiQueryParser do end end - defp parse_dimensions(dimensions) when is_list(dimensions) do + def parse_dimensions(dimensions) when is_list(dimensions) do parse_list( dimensions, &parse_dimension_entry(&1, "Invalid dimensions '#{i(dimensions)}'") ) end - defp parse_dimensions(nil), do: {:ok, []} + def parse_dimensions(nil), do: {:ok, []} def parse_order_by(order_by) when is_list(order_by) do parse_list(order_by, &parse_order_by_entry/1) diff --git a/lib/plausible/stats/dashboard/query_parser.ex b/lib/plausible/stats/dashboard/query_parser.ex index 641ddbc3e3ed..12f8d03ce467 100644 --- a/lib/plausible/stats/dashboard/query_parser.ex +++ b/lib/plausible/stats/dashboard/query_parser.ex @@ -14,19 +14,22 @@ defmodule Plausible.Stats.Dashboard.QueryParser do @valid_comparison_shorthand_keys Map.keys(@valid_comparison_shorthands) - def parse(params) do + def parse(params, opts \\ []) do with {:ok, input_date_range} <- parse_input_date_range(params), {:ok, relative_date} <- parse_relative_date(params), - {:ok, filters} <- parse_filters(params), + {:ok, dimensions} <- ApiQueryParser.parse_dimensions(params["dimensions"]), + {:ok, filters} <- ApiQueryParser.parse_filters(params["filters"]), {:ok, metrics} <- parse_metrics(params), {:ok, include} <- parse_include(params) do {:ok, ParsedQueryParams.new!(%{ input_date_range: input_date_range, relative_date: relative_date, + dimensions: dimensions, filters: filters, metrics: metrics, - include: include + include: include, + now: Keyword.get(opts, :now) })} end end @@ -78,6 +81,8 @@ defmodule Plausible.Stats.Dashboard.QueryParser do compare: compare, compare_match_day_of_week: params["include"]["compare_match_day_of_week"] == true, time_labels: params["include"]["time_labels"] == true, + partial_time_labels: params["include"]["partial_time_labels"] == true, + present_index: params["include"]["present_index"] == true, trim_relative_date_range: true, drop_unavailable_time_on_page: true }} @@ -112,8 +117,4 @@ defmodule Plausible.Stats.Dashboard.QueryParser do defp parse_include_compare(_) do {:ok, nil} end - - defp parse_filters(%{"filters" => filters}) when is_list(filters) do - Plausible.Stats.ApiQueryParser.parse_filters(filters) - end end diff --git a/lib/plausible/stats/query_builder.ex b/lib/plausible/stats/query_builder.ex index 05c91c65f272..f97aaba0baaf 100644 --- a/lib/plausible/stats/query_builder.ex +++ b/lib/plausible/stats/query_builder.ex @@ -510,11 +510,11 @@ defmodule Plausible.Stats.QueryBuilder do message: "Metric `#{metric}` cannot be queried with a filter on `event:page`." }} - length(query.dimensions) > 0 -> + Enum.any?(query.dimensions, &(not Time.time_dimension?(&1))) -> {:error, %QueryError{ code: :invalid_metrics, - message: "Metric `#{metric}` cannot be queried with `dimensions`." + message: "Metric `#{metric}` cannot be queried with non-time dimensions." }} true -> diff --git a/lib/plausible/stats/query_include.ex b/lib/plausible/stats/query_include.ex index aaf6c5b124f1..beff278dfc46 100644 --- a/lib/plausible/stats/query_include.ex +++ b/lib/plausible/stats/query_include.ex @@ -4,6 +4,13 @@ defmodule Plausible.Stats.QueryInclude do defstruct imports: false, imports_meta: false, time_labels: false, + # `time_label_result_indices` is a convenience for our main graph component. It + # is not yet ready for a public API release because it should also account for + # breakdowns by multiple dimensions (time + non-time). Also, at this point it is + # still unclear whether `time_labels` will stay in the public API or not. + time_label_result_indices: false, + present_index: false, + partial_time_labels: false, total_rows: false, trim_relative_date_range: false, compare: nil, @@ -18,6 +25,9 @@ defmodule Plausible.Stats.QueryInclude do imports: boolean(), imports_meta: boolean(), time_labels: boolean(), + time_label_result_indices: boolean(), + present_index: boolean(), + partial_time_labels: boolean(), total_rows: boolean(), trim_relative_date_range: boolean(), compare: diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex index 66e59ca7f5ce..92de19441a0e 100644 --- a/lib/plausible/stats/query_result.ex +++ b/lib/plausible/stats/query_result.ex @@ -11,6 +11,7 @@ defmodule Plausible.Stats.QueryResult do alias Plausible.Stats.{Query, QueryRunner, Filters} defstruct results: [], + comparison_results: nil, meta: %{}, query: nil @@ -43,10 +44,11 @@ defmodule Plausible.Stats.QueryResult do `results` should already-built by Plausible.Stats.QueryRunner """ - def from(%QueryRunner{results: results} = runner) do + def from(%QueryRunner{results: results, comparison_results: comparison_results} = runner) do struct!( __MODULE__, results: results, + comparison_results: comparison_results, meta: meta(runner) |> Jason.OrderedObject.new(), query: query(runner) |> Jason.OrderedObject.new() ) @@ -56,7 +58,12 @@ defmodule Plausible.Stats.QueryResult do %{} |> add_imports_meta(runner.main_query) |> add_metric_warnings_meta(runner.main_query) - |> add_time_labels_meta(runner.main_query) + |> add_time_labels_meta(runner) + |> add_time_labels_result_indices_meta(runner) + |> add_comparison_time_labels_meta(runner) + |> add_comparison_time_label_result_indices_meta(runner) + |> add_present_index_meta(runner.main_query) + |> add_partial_time_labels_meta(runner.main_query) |> add_total_rows_meta(runner.main_query, runner.total_rows) |> Enum.sort_by(&elem(&1, 0)) end @@ -85,7 +92,7 @@ defmodule Plausible.Stats.QueryResult do end end - defp add_time_labels_meta(meta, query) do + defp add_time_labels_meta(meta, %QueryRunner{main_query: query}) do if query.include.time_labels do Map.put(meta, :time_labels, Plausible.Stats.Time.time_labels(query)) else @@ -93,6 +100,73 @@ defmodule Plausible.Stats.QueryResult do end end + defp add_comparison_time_labels_meta(meta, %QueryRunner{main_query: query} = runner) do + if query.include.time_labels && query.include.compare do + Map.put( + meta, + :comparison_time_labels, + Plausible.Stats.Time.time_labels(runner.comparison_query) + ) + else + meta + end + end + + defp add_time_labels_result_indices_meta(meta, %QueryRunner{main_query: query} = runner) do + time_labels = meta[:time_labels] + + if query.include.time_label_result_indices and is_list(time_labels) do + Map.put( + meta, + :time_label_result_indices, + result_indices_for_time_labels(time_labels, runner.main_results) + ) + else + meta + end + end + + defp add_comparison_time_label_result_indices_meta( + meta, + %QueryRunner{main_query: query} = runner + ) do + comp_time_labels = meta[:comparison_time_labels] + + if query.include.time_label_result_indices and is_list(comp_time_labels) do + Map.put( + meta, + :comparison_time_label_result_indices, + result_indices_for_time_labels(comp_time_labels, runner.comparison_results) + ) + else + meta + end + end + + defp add_present_index_meta(meta, query) do + time_labels = meta[:time_labels] + + if query.include.present_index and is_list(time_labels) do + Map.put(meta, :present_index, Plausible.Stats.Time.present_index(time_labels, query)) + else + meta + end + end + + defp add_partial_time_labels_meta(meta, query) do + time_labels = meta[:time_labels] + + if query.include.partial_time_labels and is_list(time_labels) do + Map.put( + meta, + :partial_time_labels, + Plausible.Stats.Time.partial_time_labels(time_labels, query) + ) + else + meta + end + end + defp add_total_rows_meta(meta, query, total_rows) do if query.include.total_rows do Map.put(meta, :total_rows, total_rows) @@ -209,6 +283,15 @@ defmodule Plausible.Stats.QueryResult do defp metric_warning(_metric, _query), do: nil + defp result_indices_for_time_labels(time_labels, results_list) do + index_lookup_map = + results_list + |> Enum.with_index() + |> Map.new(fn {%{dimensions: [dim]}, idx} -> {dim, idx} end) + + Enum.map(time_labels, &Map.get(index_lookup_map, &1)) + end + defp to_iso8601(datetime, timezone) do datetime |> DateTime.shift_zone!(timezone) @@ -217,8 +300,25 @@ defmodule Plausible.Stats.QueryResult do end defimpl Jason.Encoder, for: Plausible.Stats.QueryResult do - def encode(%Plausible.Stats.QueryResult{results: results, meta: meta, query: query}, opts) do - Jason.OrderedObject.new(results: results, meta: meta, query: query) + def encode( + %Plausible.Stats.QueryResult{ + results: results, + comparison_results: comparison_results, + meta: meta, + query: query + }, + opts + ) do + if comparison_results do + Jason.OrderedObject.new( + results: results, + comparison_results: comparison_results, + meta: meta, + query: query + ) + else + Jason.OrderedObject.new(results: results, meta: meta, query: query) + end |> Jason.Encoder.encode(opts) end end diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex index 8088d54cafea..626289099218 100644 --- a/lib/plausible/stats/query_runner.ex +++ b/lib/plausible/stats/query_runner.ex @@ -87,27 +87,60 @@ defmodule Plausible.Stats.QueryRunner do end end - defp get_time_lookup(query, comparison_query) do - if Time.time_dimension(query) && comparison_query do - Enum.zip( - Time.time_labels(query), - Time.time_labels(comparison_query) - ) - |> Map.new() - else - %{} + # Assembles the final results, optionally attaching comparison data. + # + # Without a comparison, main results are returned as-is and comparison_results + # is nil. + # + # With comparisons, timeseries and non-time-dimension breakdowns are handled + # separately because they have fundamentally different shapes: + # + # - Non-time breakdowns (e.g. by page, source) return one row per dimension + # group. The comparison query is filtered to the same set of dimension + # values as the main query, so every comparison result is guaranteed to + # have a matching main result. Comparison data is merged inline into each + # result row; comparison_results is nil. + # + # - Timeseries (single "time:*" dimension) keep results and comparison_results + # as separate lists of only non-empty rows. Each comparison row carries a + # `change` field computed against the positionally-aligned original bucket + # (or nil when there is no corresponding original bucket). + defp build_results_list(%__MODULE__{main_query: query, main_results: main_results} = runner) do + case {query.include.compare, query.dimensions} do + {nil, _dimensions} -> + struct!(runner, + results: main_results, + comparison_results: nil + ) + + {_non_nil_compare, ["time:" <> _]} -> + struct!(runner, + results: main_results, + comparison_results: build_comparison_results(runner) + ) + + {_non_nil_compare, _dimensions} -> + struct!(runner, + results: merge_with_comparison_results(main_results, runner), + comparison_results: nil + ) end end - defp build_results_list(%__MODULE__{main_query: query, main_results: main_results} = runner) do - results = - case query.dimensions do - ["time:" <> _] -> main_results |> add_empty_timeseries_rows(runner) - _ -> main_results - end - |> merge_with_comparison_results(runner) - - struct!(runner, results: results) + defp build_comparison_results(%__MODULE__{main_query: query} = runner) do + main_map = index_by_dimensions(runner.main_results) + + comp_label_to_main_label = + Enum.zip(Time.time_labels(runner.comparison_query), Time.time_labels(query)) + |> Map.new() + + Enum.map(runner.comparison_results, fn %{dimensions: [comp_label]} = comp_row -> + main_label = Map.get(comp_label_to_main_label, comp_label) + main_metrics = main_label && Map.get(main_map, [main_label]) + change = calculate_metric_changes(query, main_metrics, comp_row.metrics) + + Map.put(comp_row, :change, change) + end) end defp execute_query(query, site) do @@ -181,75 +214,36 @@ defmodule Plausible.Stats.QueryRunner do |> Enum.at(goal_index - 1) end - # Special case: If comparison and single time dimension, add 0 rows - otherwise - # comparisons would not be shown for timeseries with 0 values. - defp add_empty_timeseries_rows(results_list, %__MODULE__{main_query: query}) - when not is_nil(query.include.compare) do - indexed_results = index_by_dimensions(results_list) - - empty_timeseries_rows = - Time.time_labels(query) - |> Enum.reject(fn dimension_value -> Map.has_key?(indexed_results, [dimension_value]) end) - |> Enum.map(fn dimension_value -> - %{ - metrics: empty_metrics(query, [dimension_value]), - dimensions: [dimension_value] - } - end) - - results_list ++ empty_timeseries_rows - end - - defp add_empty_timeseries_rows(results_list, _), do: results_list - defp merge_with_comparison_results(results_list, runner) do - comparison_map = (runner.comparison_results || []) |> index_by_dimensions() - time_lookup = get_time_lookup(runner.main_query, runner.comparison_query) - - Enum.map( - results_list, - &add_comparison_results(&1, runner.main_query, comparison_map, time_lookup) - ) + comparison_map = index_by_dimensions(runner.comparison_results) + Enum.map(results_list, &add_comparison_results(&1, runner.main_query, comparison_map)) end - defp add_comparison_results(row, query, comparison_map, time_lookup) - when not is_nil(query.include.compare) do - dimensions = get_comparison_dimensions(row.dimensions, query, time_lookup) - comparison_metrics = get_comparison_metrics(comparison_map, dimensions, query) + defp add_comparison_results(row, query, comparison_map) do + comparison_metrics = metrics_for_dimension_group(comparison_map, row.dimensions, query) change = Enum.zip([query.metrics, row.metrics, comparison_metrics]) - |> Enum.map(fn {metric, metric_value, comparison_value} -> - Compare.calculate_change(metric, comparison_value, metric_value) + |> Enum.map(fn {metric, main_value, comp_value} -> + Compare.calculate_change(metric, comp_value, main_value) end) Map.merge(row, %{ comparison: %{ - dimensions: dimensions, + dimensions: row.dimensions, metrics: comparison_metrics, change: change } }) end - defp add_comparison_results(row, _, _, _), do: row - - defp get_comparison_dimensions(dimensions, query, time_lookup) do - query.dimensions - |> Enum.zip(dimensions) - |> Enum.map(fn - {"time:" <> _, value} -> time_lookup[value] - {_, value} -> value - end) - end - defp index_by_dimensions(results_list) do results_list |> Map.new(fn entry -> {entry.dimensions, entry.metrics} end) end - defp get_comparison_metrics(comparison_map, dimensions, query) do - Map.get_lazy(comparison_map, dimensions, fn -> empty_metrics(query, dimensions) end) + defp metrics_for_dimension_group(lookup_map, dimensions, query) do + Map.get_lazy(lookup_map, dimensions, fn -> empty_metrics(query, dimensions) end) end defp empty_metrics(query, dimensions) do @@ -257,6 +251,15 @@ defmodule Plausible.Stats.QueryRunner do |> Enum.map(fn metric -> Metrics.default_value(metric, query, dimensions) end) end + defp calculate_metric_changes(query, main_metrics, comparison_metrics) do + if main_metrics do + Enum.zip([query.metrics, main_metrics, comparison_metrics]) + |> Enum.map(fn {metric, main_value, comp_value} -> + Compare.calculate_change(metric, comp_value, main_value) + end) + end + end + defp total_rows([]), do: 0 defp total_rows([first_row | _rest]), do: first_row.total_rows end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index c48ae9e0d09f..782e01e7a905 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -12,7 +12,7 @@ defmodule Plausible.Stats.SQL.Expression do import Ecto.Query - alias Plausible.Stats.{Query, Filters, SQL} + alias Plausible.Stats.{Query, Filters, SQL, Time} @no_ref "Direct / None" @no_channel "Direct" @@ -40,12 +40,39 @@ defmodule Plausible.Stats.SQL.Expression do end end + def select_dimension(q, key, "time:month", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + + select_merge_as(q, [t], %{ + key => + fragment( + "toStartOfMonth(toTimeZone(least(?, ?), ?))", + t.timestamp, + ^last_datetime, + ^query.timezone + ) + }) + end + def select_dimension(q, key, "time:month", _table, query) do select_merge_as(q, [t], %{ key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone) }) end + def select_dimension(q, key, "time:week", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + date_range = Query.date_range(query) + + select_merge_as(q, [t], %{ + key => + weekstart_not_before( + to_timezone(fragment("least(?, ?)", t.timestamp, ^last_datetime), ^query.timezone), + ^date_range.first + ) + }) + end + def select_dimension(q, key, "time:week", _table, query) do date_range = Query.date_range(query) @@ -58,6 +85,20 @@ defmodule Plausible.Stats.SQL.Expression do }) end + def select_dimension(q, key, "time:day", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + + select_merge_as(q, [t], %{ + key => + fragment( + "toDate(toTimeZone(least(?, ?), ?))", + t.timestamp, + ^last_datetime, + ^query.timezone + ) + }) + end + def select_dimension(q, key, "time:day", _table, query) do select_merge_as(q, [t], %{ key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone) @@ -69,12 +110,20 @@ defmodule Plausible.Stats.SQL.Expression do # timezone-aware. This means that for e.g. Asia/Katmandu (GMT+5:45) # to work, we divide time into 15-minute buckets and later combine these # via toStartOfHour + {first_datetime, last_datetime} = Time.utc_boundaries(query) + q |> join(:inner, [s], time_slot in time_slots(query, 15 * 60), as: :time_slot, hints: "ARRAY", on: true ) + |> where( + [s, time_slot: ts], + fragment("toStartOfHour(?)", ts) >= + fragment("toStartOfHour(toTimeZone(?, ?))", ^first_datetime, ^query.timezone) and + fragment("toStartOfHour(?)", ts) <= ^last_datetime + ) |> select_merge_as([s, time_slot: time_slot], %{ key => fragment("toStartOfHour(?)", time_slot) }) @@ -89,12 +138,19 @@ defmodule Plausible.Stats.SQL.Expression do # :NOTE: This is not exposed in Query APIv2 def select_dimension(q, key, "time:minute", :sessions, query) when query.smear_session_metrics do + {first_datetime, last_datetime} = Time.utc_boundaries(query) + q |> join(:inner, [s], time_slot in time_slots(query, 60), as: :time_slot, hints: "ARRAY", on: true ) + |> where( + [s, time_slot: ts], + ts >= fragment("toStartOfMinute(toTimeZone(?, ?))", ^first_datetime, ^query.timezone) and + ts <= ^last_datetime + ) |> select_merge_as([s, time_slot: time_slot], %{ key => fragment("?", time_slot) }) diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex index 91758784f818..7d770172300b 100644 --- a/lib/plausible/stats/time.ex +++ b/lib/plausible/stats/time.ex @@ -120,6 +120,64 @@ defmodule Plausible.Stats.Time do |> Enum.map(&format_datetime/1) end + def partial_time_labels(time_labels, query) do + case time_dimension(query) do + "time:week" -> + date_range = Query.date_range(query) + partial_labels(time_labels, date_range, &Date.beginning_of_week/1, &Date.end_of_week/1) + + "time:month" -> + date_range = Query.date_range(query) + partial_labels(time_labels, date_range, &Date.beginning_of_month/1, &Date.end_of_month/1) + + _ -> + nil + end + end + + defp partial_labels(time_labels, date_range, start_fn, end_fn) do + Enum.filter(time_labels, fn label -> + case Date.from_iso8601(label) do + {:ok, date} -> + start_in_range = Enum.member?(date_range, start_fn.(date)) + end_in_range = Enum.member?(date_range, end_fn.(date)) + not start_in_range or not end_in_range + + _ -> + false + end + end) + end + + def present_index(time_labels, query) do + now = DateTime.now!(query.timezone) + + current_label = + case time_dimension(query) do + "time:month" -> + DateTime.to_date(now) + |> Date.beginning_of_month() + |> Date.to_string() + + "time:week" -> + DateTime.to_date(now) + |> date_or_weekstart(Query.date_range(query)) + |> Date.to_string() + + "time:day" -> + DateTime.to_date(now) + |> Date.to_string() + + "time:hour" -> + Calendar.strftime(now, "%Y-%m-%d %H:00:00") + + "time:minute" -> + Calendar.strftime(now, "%Y-%m-%d %H:%M:00") + end + + Enum.find_index(time_labels, &(&1 == current_label)) + end + def date_or_weekstart(date, date_range) do weekstart = Date.beginning_of_week(date) diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 12f662ef17d3..0f5de54039ad 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -7,7 +7,7 @@ defmodule Plausible.Stats.Timeseries do use Plausible use Plausible.ClickhouseRepo - alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time, QueryOptimizer} + alias Plausible.Stats.{Query, QueryRunner, Metrics, Time, QueryOptimizer} @time_dimension %{ "month" => "time:month", @@ -28,17 +28,10 @@ defmodule Plausible.Stats.Timeseries do ) |> QueryOptimizer.optimize() - comparison_query = - if(query.include.compare, - do: Comparisons.get_comparison_query(query), - else: nil - ) - query_result = QueryRunner.run(site, query) { - build_result(query_result, query, fn entry -> entry end), - build_result(query_result, comparison_query, fn entry -> entry.comparison end), + build_result(query_result, query), query_result.meta } end @@ -47,23 +40,21 @@ defmodule Plausible.Stats.Timeseries do # Given a query result, build a legacy timeseries result # Format is %{ date => %{ date: date_string, [metric] => value } } with a bunch of special cases for the UI - defp build_result(query_result, %Query{} = query, extract_entry) do + defp build_result(query_result, %Query{} = query) do query_result.results - |> Enum.map(&extract_entry.(&1)) - |> Enum.map(fn %{dimensions: [time_dimension_value], metrics: metrics} -> - metrics_map = Enum.zip(query.metrics, metrics) |> Map.new() - - { - time_dimension_value, - Map.put(metrics_map, :date, time_dimension_value) - } + |> Enum.map(fn + %{dimensions: [time_dimension_value], metrics: metrics} -> + metrics_map = Enum.zip(query.metrics, metrics) |> Map.new() + + { + time_dimension_value, + Map.put(metrics_map, :date, time_dimension_value) + } end) |> Map.new() |> add_labels(query) end - defp build_result(_, _, _), do: nil - defp add_labels(results_map, query) do query |> Time.time_labels() diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 77ad190aece5..49d7a4786ebb 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -257,7 +257,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do :ok <- validate_filters(site, query.filters), {:ok, metrics} <- parse_and_validate_metrics(params, query), :ok <- ensure_custom_props_access(site, query) do - {results, _, meta} = Plausible.Stats.timeseries(site, query, metrics) + {results, meta} = Plausible.Stats.timeseries(site, query, metrics) payload = case meta[:imports_warning] do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 9d471c664b82..42e7c62d41ba 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -10,7 +10,6 @@ defmodule PlausibleWeb.Api.StatsController do Query, Comparisons, Filters, - Time, TableDecider, TimeOnPage, Dashboard, @@ -31,182 +30,23 @@ defmodule PlausibleWeb.Api.StatsController do def query(conn, params) do site = conn.assigns.site + now = conn.private[:now] - with {:ok, %ParsedQueryParams{} = params} <- Dashboard.QueryParser.parse(params), + with {:ok, %ParsedQueryParams{} = params} <- Dashboard.QueryParser.parse(params, now: now), {:ok, %Query{} = query} <- QueryBuilder.build(site, params, debug_metadata(conn)) do + query = + if query.include.time_labels do + Query.set_include(query, :time_label_result_indices, true) + else + query + end + json(conn, Plausible.Stats.query(site, query)) else {:error, %QueryError{message: message}} -> bad_request(conn, message) end end - @doc """ - Returns a time-series based on given parameters. - - ## Parameters - - This API accepts the following parameters: - - * `period` - x-axis of the graph, e.g. `12mo`, `day`, `custom`. - - * `metric` - y-axis of the graph, e.g. `visits`, `visitors`, `pageviews`. - See the Stats API ["Metrics"](https://plausible.io/docs/stats-api#metrics) - section for more details. Defaults to `visitors`. - - * `interval` - granularity of the time-series data. You can think of it as - a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`, `week`, - and `month`. The default depends on the `period` parameter. Check - `Plausible.Query.from/2` for each default. - - * `filters` - optional filters to drill down data. See the Stats API - ["Filtering"](https://plausible.io/docs/stats-api#filtering) section for - more details. - - * `with_imported` - boolean indicating whether to include Google Analytics - imported data or not. Defaults to `false`. - - Full example: - ```elixir - %{ - "from" => "2021-09-06", - "interval" => "month", - "metric" => "visitors", - "period" => "custom", - "to" => "2021-12-13" - } - ``` - - ## Response - - Returns a map with the following keys: - - * `plot` - list of values for the requested metric representing the y-axis - of the graph. - - * `labels` - list of date times representing the x-axis of the graph. - - * `present_index` - index of the element representing the current date in - `labels` and `plot` lists. - - * `interval` - the interval used for querying. - - * `includes_imported` - boolean indicating whether imported data - was queried or not. - - * `full_intervals` - map of dates indicating whether the interval has been - cut off by the requested date range or not. For example, if looking at a - month week-by-week, some weeks may be cut off by the month boundaries. - It's useful to adjust the graph display slightly in case the interval is - not 'full' so that the user understands why the numbers might be lower for - those partial periods. - - Full example: - ```elixir - %{ - "full_intervals" => %{ - "2021-09-01" => false, - "2021-10-01" => true, - "2021-11-01" => true, - "2021-12-01" => false - }, - "interval" => "month", - "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"], - "plot" => [0, 0, 0, 0], - "present_index" => nil, - "includes_imported" => false - } - ``` - - """ - def main_graph(conn, params) do - site = conn.assigns[:site] - now = conn.private[:now] - - with {:ok, dates} <- parse_date_params(params), - :ok <- validate_interval(params), - :ok <- validate_interval_granularity(site, params, dates), - params <- realtime_period_to_30m(params), - query = Query.from(site, params, debug_metadata: debug_metadata(conn), now: now), - query <- Query.set_include(query, :trim_relative_date_range, true), - {:ok, metric} <- parse_and_validate_graph_metric(params, query) do - {timeseries_result, comparison_result, _meta} = Stats.timeseries(site, query, [metric]) - - labels = label_timeseries(timeseries_result, comparison_result) - present_index = present_index_for(site, query, labels) - full_intervals = build_full_intervals(query, labels) - - json(conn, %{ - metric: metric, - plot: plot_timeseries(timeseries_result, metric), - labels: labels, - comparison_plot: comparison_result && plot_timeseries(comparison_result, metric), - comparison_labels: comparison_result && label_timeseries(comparison_result, nil), - present_index: present_index, - full_intervals: full_intervals - }) - else - {:error, message} when is_binary(message) -> bad_request(conn, message) - end - end - - defp plot_timeseries(timeseries, metric) do - Enum.map(timeseries, & &1[metric]) - end - - defp label_timeseries(main_result, nil) do - Enum.map(main_result, & &1.date) - end - - @blank_value "__blank__" - defp label_timeseries(main_result, comparison_result) do - blanks_to_fill = Enum.count(comparison_result) - Enum.count(main_result) - - if blanks_to_fill > 0 do - blanks = List.duplicate(@blank_value, blanks_to_fill) - Enum.map(main_result, & &1.date) ++ blanks - else - Enum.map(main_result, & &1.date) - end - end - - defp build_full_intervals( - %Query{interval: "week"} = query, - labels - ) do - date_range = Query.date_range(query) - build_intervals(labels, date_range, &Date.beginning_of_week/1, &Date.end_of_week/1) - end - - defp build_full_intervals( - %Query{interval: "month"} = query, - labels - ) do - date_range = Query.date_range(query) - build_intervals(labels, date_range, &Date.beginning_of_month/1, &Date.end_of_month/1) - end - - defp build_full_intervals(_query, _labels) do - nil - end - - def build_intervals(labels, date_range, start_fn, end_fn) do - for label <- labels, into: %{} do - case Date.from_iso8601(label) do - {:ok, date} -> - interval_start = start_fn.(date) - interval_end = end_fn.(date) - - within_interval? = - Enum.member?(date_range, interval_start) && Enum.member?(date_range, interval_end) - - {label, within_interval?} - - _ -> - {label, false} - end - end - end - def top_stats(conn, params) do site = conn.assigns[:site] @@ -257,52 +97,6 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp present_index_for(site, query, dates) do - case query.interval do - "hour" -> - current_date = - DateTime.now!(site.timezone) - |> Calendar.strftime("%Y-%m-%d %H:00:00") - - Enum.find_index(dates, &(&1 == current_date)) - - "day" -> - current_date = - DateTime.now!(site.timezone) - |> DateTime.to_date() - |> Date.to_string() - - Enum.find_index(dates, &(&1 == current_date)) - - "week" -> - date_range = Query.date_range(query) - - current_date = - DateTime.now!(site.timezone) - |> DateTime.to_date() - |> Time.date_or_weekstart(date_range) - |> Date.to_string() - - Enum.find_index(dates, &(&1 == current_date)) - - "month" -> - current_date = - DateTime.now!(site.timezone) - |> DateTime.to_date() - |> Date.beginning_of_month() - |> Date.to_string() - - Enum.find_index(dates, &(&1 == current_date)) - - "minute" -> - current_date = - DateTime.now!(site.timezone) - |> Calendar.strftime("%Y-%m-%d %H:%M:00") - - Enum.find_index(dates, &(&1 == current_date)) - end - end - defp fetch_top_stats(site, query) do goal_filter? = toplevel_goal_filter?(query) @@ -1694,75 +1488,6 @@ defmodule PlausibleWeb.Api.StatsController do end) end - defp validate_interval(params) do - with %{"interval" => interval} <- params, - true <- Plausible.Stats.Interval.valid?(interval) do - :ok - else - %{} -> - :ok - - false -> - values = Enum.join(Plausible.Stats.Interval.list(), ", ") - {:error, "Invalid value for interval. Accepted values are: #{values}"} - end - end - - defp validate_interval_granularity(site, params, dates) do - case params do - %{"interval" => interval, "period" => "custom", "from" => _, "to" => _} -> - if Plausible.Stats.Interval.valid_for_period?("custom", interval, - site: site, - from: dates["from"], - to: dates["to"] - ) do - :ok - else - {:error, - "Invalid combination of interval and period. Custom ranges over 12 months must come with greater granularity, e.g. `period=custom,interval=week`"} - end - - %{"interval" => interval, "period" => period} -> - if Plausible.Stats.Interval.valid_for_period?(period, interval, site: site) do - :ok - else - {:error, - "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"} - end - - _ -> - :ok - end - end - - defp parse_and_validate_graph_metric(params, query) do - metric = - case params["metric"] do - nil -> :visitors - "conversions" -> :visitors - m -> Plausible.Stats.Metrics.from_string!(m) - end - - requires_goal_filter? = metric in [:conversion_rate, :events] - has_goal_filter? = toplevel_goal_filter?(query) - - requires_page_filter? = metric == :scroll_depth - - has_page_filter? = - Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) - - cond do - requires_goal_filter? and not has_goal_filter? -> - {:error, "Metric `#{metric}` can only be queried with a goal filter"} - - requires_page_filter? and not has_page_filter? -> - {:error, "Metric `#{metric}` can only be queried with a page filter"} - - true -> - {:ok, metric} - end - end - defp bad_request(conn, message, extra \\ %{}) do payload = Map.merge(extra, %{error: message}) diff --git a/test/plausible/stats/dashboard/query_parser_test.exs b/test/plausible/stats/dashboard/query_parser_test.exs index d7704ec44660..32b08eaa8dbc 100644 --- a/test/plausible/stats/dashboard/query_parser_test.exs +++ b/test/plausible/stats/dashboard/query_parser_test.exs @@ -249,12 +249,20 @@ defmodule Plausible.Stats.Dashboard.QueryParserTest do params = Map.merge(@base_params, %{"metrics" => []}) assert {:error, %QueryError{code: :invalid_metrics}} = parse(params) end + end - test "now can't be fixed externally" do + describe "fixing now" do + test "now can't be fixed externally via params" do params = Map.merge(@base_params, %{"now" => "2026-02-17T10:08:52.272894Z"}) {:ok, parsed} = parse(params) assert parsed.now == nil end + + test "now can be fixed as an optional extra argument to parse/2" do + {:ok, parsed} = parse(@base_params, now: ~U[2026-02-17 10:08:00Z]) + + assert parsed.now == ~U[2026-02-17 10:08:00Z] + end end end diff --git a/test/plausible/stats/query/query_comparisons_test.exs b/test/plausible/stats/query/query_comparisons_test.exs index 67d8f9ed2f0a..cb0b62a5f7a1 100644 --- a/test/plausible/stats/query/query_comparisons_test.exs +++ b/test/plausible/stats/query/query_comparisons_test.exs @@ -48,72 +48,17 @@ defmodule Plausible.Stats.QueryComparisonsTest do include: %QueryInclude{compare: :previous_period} }) - assert %Stats.QueryResult{results: results} = Stats.query(site, query) + assert %Stats.QueryResult{results: results, comparison_results: comparison_results} = + Stats.query(site, query) assert results == [ - %{ - dimensions: ["2021-01-07"], - metrics: [1], - comparison: %{ - dimensions: ["2020-12-31"], - metrics: [0], - change: [100] - } - }, - %{ - dimensions: ["2021-01-08"], - metrics: [1], - comparison: %{ - dimensions: ["2021-01-01"], - metrics: [2], - change: [-50] - } - }, - %{ - dimensions: ["2021-01-09"], - metrics: [0], - comparison: %{ - dimensions: ["2021-01-02"], - metrics: [0], - change: [0] - } - }, - %{ - dimensions: ["2021-01-10"], - metrics: [0], - comparison: %{ - dimensions: ["2021-01-03"], - metrics: [0], - change: [0] - } - }, - %{ - dimensions: ["2021-01-11"], - metrics: [0], - comparison: %{ - dimensions: ["2021-01-04"], - metrics: [0], - change: [0] - } - }, - %{ - dimensions: ["2021-01-12"], - metrics: [0], - comparison: %{ - dimensions: ["2021-01-05"], - metrics: [0], - change: [0] - } - }, - %{ - dimensions: ["2021-01-13"], - metrics: [0], - comparison: %{ - dimensions: ["2021-01-06"], - metrics: [1], - change: [-100] - } - } + %{dimensions: ["2021-01-07"], metrics: [1]}, + %{dimensions: ["2021-01-08"], metrics: [1]} + ] + + assert comparison_results == [ + %{dimensions: ["2021-01-01"], metrics: [2], change: [-50]}, + %{dimensions: ["2021-01-06"], metrics: [1], change: nil} ] end @@ -135,34 +80,23 @@ defmodule Plausible.Stats.QueryComparisonsTest do query2 = Stats.Query.set_include(query1, :compare_match_day_of_week, true) - assert %Stats.QueryResult{results: results1} = Stats.query(site, query1) - assert %Stats.QueryResult{results: results2} = Stats.query(site, query2) + assert %Stats.QueryResult{results: results1, meta: meta1} = Stats.query(site, query1) + assert %Stats.QueryResult{results: results2, meta: meta2} = Stats.query(site, query2) + + assert results1 == [] + assert results2 == [] - assert results1 == results2 + assert meta1[:time_labels] == meta2[:time_labels] expected_first_date = today |> Date.shift(day: -28) |> Date.to_iso8601() expected_last_date = today |> Date.shift(day: -1) |> Date.to_iso8601() expected_comparison_first_date = today |> Date.shift(day: -56) |> Date.to_iso8601() expected_comparison_last_date = today |> Date.shift(day: -29) |> Date.to_iso8601() - assert %{ - dimensions: [actual_first_date], - comparison: %{ - dimensions: [actual_comparison_first_date] - } - } = List.first(results1) - - assert %{ - dimensions: [actual_last_date], - comparison: %{ - dimensions: [actual_comparison_last_date] - } - } = List.last(results1) - - assert actual_first_date == expected_first_date - assert actual_last_date == expected_last_date - assert actual_comparison_first_date == expected_comparison_first_date - assert actual_comparison_last_date == expected_comparison_last_date + assert List.first(meta1[:time_labels]) == expected_first_date + assert List.last(meta1[:time_labels]) == expected_last_date + assert List.first(meta1[:comparison_time_labels]) == expected_comparison_first_date + assert List.last(meta1[:comparison_time_labels]) == expected_comparison_last_date end test "timeseries last 91d period in year_over_year comparison", %{ @@ -189,40 +123,33 @@ defmodule Plausible.Stats.QueryComparisonsTest do now: ~U[2022-07-01 14:00:00Z] }) - assert %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query) + assert %Stats.QueryResult{ + results: results, + comparison_results: comparison_results, + meta: meta + } = Stats.query(site, query) time_labels = meta[:time_labels] + comparison_time_labels = meta[:comparison_time_labels] + + assert length(time_labels) == 91 + assert length(comparison_time_labels) == 91 assert "2022-04-01" = List.first(time_labels) assert "2022-04-05" = Enum.at(time_labels, 4) assert "2022-06-30" = List.last(time_labels) - assert %{ - dimensions: ["2022-04-01"], - metrics: [1], - comparison: %{ - dimensions: ["2021-04-01"], - metrics: [2] - } - } = Enum.find(results, &(&1[:dimensions] == ["2022-04-01"])) - - assert %{ - dimensions: ["2022-04-05"], - metrics: [1], - comparison: %{ - dimensions: ["2021-04-05"], - metrics: [2] - } - } = Enum.find(results, &(&1[:dimensions] == ["2022-04-05"])) - - assert %{ - dimensions: ["2022-06-30"], - metrics: [1], - comparison: %{ - dimensions: ["2021-06-30"], - metrics: [1] - } - } = Enum.find(results, &(&1[:dimensions] == ["2022-06-30"])) + assert results == [ + %{dimensions: ["2022-04-01"], metrics: [1]}, + %{dimensions: ["2022-04-05"], metrics: [1]}, + %{dimensions: ["2022-06-30"], metrics: [1]} + ] + + assert comparison_results == [ + %{dimensions: ["2021-04-01"], metrics: [2], change: [-50]}, + %{dimensions: ["2021-04-05"], metrics: [2], change: [-50]}, + %{dimensions: ["2021-06-30"], metrics: [1], change: [0]} + ] end test "dimensional comparison with low limit", %{site: site} do diff --git a/test/plausible/stats/query/query_parse_and_build_test.exs b/test/plausible/stats/query/query_parse_and_build_test.exs index ef705cc883d1..eb4320b220db 100644 --- a/test/plausible/stats/query/query_parse_and_build_test.exs +++ b/test/plausible/stats/query/query_parse_and_build_test.exs @@ -2039,7 +2039,7 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do assert error == "Metric `views_per_visit` cannot be queried with a filter on `event:page`." end - test "fails validation with dimensions", %{site: site} do + test "fails validation with non-time dimensions", %{site: site} do params = %{ "site_id" => site.domain, "metrics" => ["views_per_visit"], @@ -2050,7 +2050,7 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do assert {:error, %QueryError{message: error}} = Query.parse_and_build(site, params, now: @now) - assert error == "Metric `views_per_visit` cannot be queried with `dimensions`." + assert error == "Metric `views_per_visit` cannot be queried with non-time dimensions." end end diff --git a/test/plausible/stats/query/query_special_metrics_test.exs b/test/plausible/stats/query/query_special_metrics_test.exs index c6a7ce48a8d7..a6c31aa3b622 100644 --- a/test/plausible/stats/query/query_special_metrics_test.exs +++ b/test/plausible/stats/query/query_special_metrics_test.exs @@ -362,27 +362,17 @@ defmodule Plausible.Stats.QuerySpecialMetricsTest do include: %QueryInclude{compare: :previous_period} }) - %Stats.QueryResult{results: results} = Stats.query(site, query) + %Stats.QueryResult{results: results, comparison_results: comparison_results} = + Stats.query(site, query) assert results == [ - %{ - dimensions: ["2021-01-03"], - metrics: [250], - comparison: %{ - dimensions: ["2021-01-01"], - metrics: [nil], - change: [nil] - } - }, - %{ - dimensions: ["2021-01-04"], - metrics: [150], - comparison: %{ - dimensions: ["2021-01-02"], - metrics: [200], - change: [-25] - } - } + %{dimensions: ["2021-01-03"], metrics: [250]}, + %{dimensions: ["2021-01-04"], metrics: [150]} + ] + + assert comparison_results == [ + %{dimensions: ["2021-01-01"], metrics: [nil], change: [nil]}, + %{dimensions: ["2021-01-02"], metrics: [200], change: [-25]} ] end end diff --git a/test/plausible/stats/query/query_test.exs b/test/plausible/stats/query/query_test.exs index 90c4671aabc8..b33c21b14c33 100644 --- a/test/plausible/stats/query/query_test.exs +++ b/test/plausible/stats/query/query_test.exs @@ -125,4 +125,292 @@ defmodule Plausible.Stats.QueryTest do ] end end + + describe "timeseries with comparisons" do + test "returns more original time range buckets than comparison buckets", + %{site: site} do + populate_stats(site, [ + # original time range + build(:pageview, user_id: 123, timestamp: ~N[2026-01-03 00:00:00]), + build(:pageview, user_id: 123, timestamp: ~N[2026-01-03 00:10:00]), + build(:pageview, timestamp: ~N[2026-01-05 00:00:00]), + # comparison time range + build(:pageview, timestamp: ~N[2025-12-16 00:00:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors, :pageviews], + input_date_range: {:date_range, ~D[2025-12-25], ~D[2026-01-06]}, + dimensions: ["time:week"], + include: %QueryInclude{ + compare: {:date_range, ~D[2025-12-12], ~D[2025-12-21]}, + time_labels: true, + time_label_result_indices: true + } + }) + + %Stats.QueryResult{results: results, comparison_results: comparison_results, meta: meta} = + Stats.query(site, query) + + assert results == [ + %{dimensions: ["2025-12-29"], metrics: [1, 2]}, + %{dimensions: ["2026-01-05"], metrics: [1, 1]} + ] + + assert comparison_results == [ + %{dimensions: ["2025-12-15"], metrics: [1, 1], change: [0, 100]} + ] + + assert meta[:time_labels] == ["2025-12-25", "2025-12-29", "2026-01-05"] + assert meta[:time_label_result_indices] == [nil, 0, 1] + assert meta[:comparison_time_labels] == ["2025-12-12", "2025-12-15"] + assert meta[:comparison_time_label_result_indices] == [nil, 0] + end + + test "can return more comparison time buckets than original time range buckets", + %{site: site} do + populate_stats(site, [ + # original time range + build(:pageview, user_id: 123, timestamp: ~N[2021-02-01 00:00:00]), + build(:pageview, user_id: 123, timestamp: ~N[2021-02-01 00:10:00]), + build(:pageview, timestamp: ~N[2021-02-01 00:00:00]), + # comparison time range + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors, :pageviews], + input_date_range: {:date_range, ~D[2021-02-01], ~D[2021-02-01]}, + dimensions: ["time:day"], + include: %QueryInclude{ + compare: {:date_range, ~D[2021-01-01], ~D[2021-01-05]}, + time_labels: true, + time_label_result_indices: true + } + }) + + %Stats.QueryResult{results: results, comparison_results: comparison_results, meta: meta} = + Stats.query(site, query) + + assert results == [ + %{dimensions: ["2021-02-01"], metrics: [2, 3]} + ] + + assert comparison_results == [ + %{dimensions: ["2021-01-01"], metrics: [2, 2], change: [0, 50]}, + %{dimensions: ["2021-01-02"], metrics: [1, 1], change: nil}, + %{dimensions: ["2021-01-04"], metrics: [1, 1], change: nil} + ] + + assert meta[:time_labels] == ["2021-02-01"] + + assert meta[:comparison_time_labels] == [ + "2021-01-01", + "2021-01-02", + "2021-01-03", + "2021-01-04", + "2021-01-05" + ] + + assert meta[:time_label_result_indices] == [0] + assert meta[:comparison_time_label_result_indices] == [0, 1, nil, 2, nil] + end + end + + describe "session smearing respects query date range boundaries" do + test "time:hour does not include buckets from outside the query range", + %{site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 23:55:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-01-02 00:10:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-02 23:55:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-03 00:10:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, ~D[2021-01-02], ~D[2021-01-02]}, + dimensions: ["time:hour"], + include: %QueryInclude{total_rows: true} + }) + + %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query) + + assert results == [ + %{dimensions: ["2021-01-02 00:00:00"], metrics: [1]}, + %{dimensions: ["2021-01-02 23:00:00"], metrics: [1]} + ] + + assert meta[:total_rows] == 2 + end + + test "time:hour does not include buckets from outside the query range (non-UTC timezone)", + %{site: site} do + # America/New_York is UTC-5 in January + site = %{site | timezone: "America/New_York"} + + populate_stats(site, [ + # 2020-12-31 23:55 in NYC (outside of query range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 04:55:00]), + # 2021-01-01 00:10 in NYC (in query range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:10:00]), + # 2021-01-01 23:55 in NYC (in query range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-02 04:55:00]), + # 2021-01-02 00:10 in NYC (outside of query range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-02 05:10:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-01-01]}, + dimensions: ["time:hour"] + }) + + %Stats.QueryResult{results: results} = Stats.query(site, query) + + assert results == [ + %{dimensions: ["2021-01-01 00:00:00"], metrics: [1]}, + %{dimensions: ["2021-01-01 23:00:00"], metrics: [1]} + ] + end + + test "time:minute does not include buckets from outside the query range", + %{site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:05:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:20:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:08:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:10:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:datetime_range, ~U[2021-01-01 00:08:00Z], ~U[2021-01-01 00:12:00Z]}, + dimensions: ["time:minute"], + include: %QueryInclude{total_rows: true} + }) + + %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query) + + assert results == [ + %{dimensions: ["2021-01-01 00:08:00"], metrics: [2]}, + %{dimensions: ["2021-01-01 00:09:00"], metrics: [2]}, + %{dimensions: ["2021-01-01 00:10:00"], metrics: [2]}, + %{dimensions: ["2021-01-01 00:11:00"], metrics: [1]}, + %{dimensions: ["2021-01-01 00:12:00"], metrics: [1]} + ] + + assert meta[:total_rows] == 5 + end + + test "time:minute does not include buckets from outside the query range (non-UTC timezone)", + %{site: site} do + # America/New_York is UTC-5 in January + site = %{site | timezone: "America/New_York"} + + populate_stats(site, [ + # 2020-12-31 23:59:00 in NYC (outside of queried range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 04:59:00]), + # 2021-01-01 00:02:00 in NYC (in queried range) + build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:02:00]), + # 2021-01-01 23:59:00 in NYC (in queried range) + build(:pageview, user_id: 2, timestamp: ~N[2021-01-02 04:59:00]), + # 2021-01-02 00:01:00 in NYC (outside of queried range) + build(:pageview, user_id: 2, timestamp: ~N[2021-01-02 05:01:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: :day, + relative_date: ~D[2021-01-01], + dimensions: ["time:minute"] + }) + + %Stats.QueryResult{results: results} = Stats.query(site, query) + + assert results == [ + %{dimensions: ["2021-01-01 00:00:00"], metrics: [1]}, + %{dimensions: ["2021-01-01 00:01:00"], metrics: [1]}, + %{dimensions: ["2021-01-01 00:02:00"], metrics: [1]}, + %{dimensions: ["2021-01-01 23:59:00"], metrics: [1]} + ] + end + + test "time:day clamps sessions extending past the query range end into the last bucket", + %{site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-01-31 23:55:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-02-01 00:05:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-01-31]}, + dimensions: ["time:day"] + }) + + %Stats.QueryResult{results: results} = Stats.query(site, query) + + # Without clamping the session would bucket to "2021-02-01" (outside range) + assert results == [ + %{dimensions: ["2021-01-31"], metrics: [1]} + ] + end + + test "time:week clamps sessions extending past the query range end into the last bucket", + %{site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-01-31 23:55:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-02-01 00:05:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-01-31]}, + dimensions: ["time:week"] + }) + + %Stats.QueryResult{results: results} = Stats.query(site, query) + + # Without clamping the session would bucket to "2021-02-01" (outside range). + # Clamped to Jan 31 23:59:59 -> toMonday(Jan 31) = Jan 25. + assert results == [ + %{dimensions: ["2021-01-25"], metrics: [1]} + ] + end + + test "time:month clamps sessions extending past the query range end into the last bucket", + %{site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-02-28 23:55:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-03-01 00:05:00]) + ]) + + {:ok, query} = + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-02-28]}, + dimensions: ["time:month"] + }) + + %Stats.QueryResult{results: results} = Stats.query(site, query) + + # Without clamping the session would bucket to "2021-03-01" (outside range). + # Clamped to Feb 28 23:59:59 -> toStartOfMonth -> Feb 1. + assert results == [ + %{dimensions: ["2021-02-01"], metrics: [1]} + ] + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs index 1418f1ca2de1..fedf806a71df 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs @@ -196,10 +196,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do } end - test "validates that metric views_per_visit cannot be used together with dimensions", %{ - conn: conn, - site: site - } do + test "validates that metric views_per_visit cannot be used together with non-time dimensions", + %{ + conn: conn, + site: site + } do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, @@ -209,7 +210,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do }) assert json_response(conn, 400) == %{ - "error" => "Metric `views_per_visit` cannot be queried with `dimensions`." + "error" => "Metric `views_per_visit` cannot be queried with non-time dimensions." } end diff --git a/test/plausible_web/controllers/api/stats_controller/authorization_test.exs b/test/plausible_web/controllers/api/stats_controller/authorization_test.exs index 9c0917f3d1f9..51cd20d71983 100644 --- a/test/plausible_web/controllers/api/stats_controller/authorization_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/authorization_test.exs @@ -24,9 +24,14 @@ defmodule PlausibleWeb.Api.StatsController.AuthorizationTest do test "returns stats for public site", %{conn: conn} do conn = init_session(conn) site = insert(:site, public: true) - conn = get(conn, "/api/stats/#{site.domain}/main-graph") - assert %{"plot" => _any} = json_response(conn, 200) + conn = + post(conn, "/api/stats/#{site.domain}/query", %{ + "date_range" => "day", + "metrics" => ["visitors"] + }) + + assert %{"results" => _} = json_response(conn, 200) end end @@ -185,16 +190,26 @@ defmodule PlausibleWeb.Api.StatsController.AuthorizationTest do test "returns stats for public site", %{conn: conn} do site = new_site(public: true) - conn = get(conn, "/api/stats/#{site.domain}/main-graph") - assert %{"plot" => _any} = json_response(conn, 200) + conn = + post(conn, "/api/stats/#{site.domain}/query", %{ + "date_range" => "day", + "metrics" => ["visitors"] + }) + + assert %{"results" => _} = json_response(conn, 200) end test "returns stats for a private site that the user owns", %{conn: conn, user: user} do site = new_site(public: false, owner: user) - conn = get(conn, "/api/stats/#{site.domain}/main-graph") - assert %{"plot" => _any} = json_response(conn, 200) + conn = + post(conn, "/api/stats/#{site.domain}/query", %{ + "date_range" => "day", + "metrics" => ["visitors"] + }) + + assert %{"results" => _} = json_response(conn, 200) end end end diff --git a/test/plausible_web/controllers/api/stats_controller/debug_metadata_test.exs b/test/plausible_web/controllers/api/stats_controller/debug_metadata_test.exs index 42e379bc639f..282d7b42ae4d 100644 --- a/test/plausible_web/controllers/api/stats_controller/debug_metadata_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/debug_metadata_test.exs @@ -5,10 +5,15 @@ defmodule PlausibleWeb.Api.StatsController.DebugMetadataTest do describe "Debug metadata for logged in requests" do setup [:create_user, :log_in] - test "for main-graph", %{conn: conn, user: user} do + test "for /api/stats/:domain/query", %{conn: conn, user: user} do domain = :rand.bytes(20) |> Base.url_encode64() site = new_site(domain: domain, owner: user) - conn = get(conn, "/api/stats/#{site.domain}/main-graph") + + conn = + post(conn, "/api/stats/#{site.domain}/query", %{ + "date_range" => "day", + "metrics" => ["visitors"] + }) assert json_response(conn, 200) @@ -24,11 +29,16 @@ defmodule PlausibleWeb.Api.StatsController.DebugMetadataTest do decoded = Jason.decode!(unparsed_log_comment) assert_matches ^strict_map(%{ - "params" => ^strict_map(%{"domain" => ^site.domain}), - "phoenix_action" => "main_graph", + "params" => + ^strict_map(%{ + "domain" => ^site.domain, + "date_range" => "day", + "metrics" => ["visitors"] + }), + "phoenix_action" => "query", "phoenix_controller" => "Elixir.PlausibleWeb.Api.StatsController", - "request_method" => "GET", - "request_path" => ^"/api/stats/#{site.domain}/main-graph", + "request_method" => "POST", + "request_path" => ^"/api/stats/#{site.domain}/query", "site_domain" => ^site.domain, "site_id" => ^site.id, "team_id" => ^team_of(user).id, diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs index bf7f30d1141c..52c7920b360d 100644 --- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs @@ -73,18 +73,24 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do "imported_visitors" ) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" - ) + params = %{ + "date_range" => "month", + "metrics" => ["visitors"], + "relative_date" => "2021-01-01", + "dimensions" => ["time:day"], + "include" => %{"imports" => true, "time_labels" => true} + } - assert %{"plot" => plot} = json_response(conn, 200) + conn = post(conn, "/api/stats/#{site.domain}/query", params) - assert Enum.count(plot) == 31 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + response = json_response(conn, 200) + + assert length(response["meta"]["time_labels"]) == 31 + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [2]} + ] end test "returns data grouped by week", %{conn: conn, site: site, import_id: import_id} do @@ -121,18 +127,24 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do "imported_visitors" ) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week" - ) + params = %{ + "date_range" => "month", + "metrics" => ["visitors"], + "relative_date" => "2021-01-01", + "dimensions" => ["time:week"], + "include" => %{"imports" => true, "time_labels" => true} + } - assert %{"plot" => plot} = json_response(conn, 200) + conn = post(conn, "/api/stats/#{site.domain}/query", params) - assert Enum.count(plot) == 5 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + response = json_response(conn, 200) + + assert length(response["meta"]["time_labels"]) == 5 + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-25"], "metrics" => [2]} + ] end test "Sources are imported", %{conn: conn, site: site, import_id: import_id} do diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 21e1c52bc8fd..c73dff886981 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -3,21 +3,47 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do @user_id Enum.random(1000..9999) - describe "GET /api/stats/main-graph - plot" do + defp do_query(conn, site, params, opts \\ []) do + now = Keyword.get(opts, :now) + + conn + |> Plug.Conn.put_private(:now, now) + |> post("/api/stats/#{site.domain}/query", params) + |> json_response(200) + end + + defp do_query_fail(conn, site, params) do + conn + |> post("/api/stats/#{site.domain}/query", params) + end + + describe "plot" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: relative_time(minute: -5)) + build(:pageview, timestamp: ~N[2024-04-02 03:20:46]) ]) - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime&metric=pageviews") + response = + do_query( + conn, + site, + %{ + "date_range" => "realtime_30m", + "metrics" => ["pageviews"], + "dimensions" => ["time:minute"], + "include" => %{"time_labels" => true} + }, + now: ~U[2024-04-02 03:27:30Z] + ) - assert %{"plot" => plot, "labels" => labels} = json_response(conn, 200) + %{"results" => results, "meta" => meta} = response - assert labels == Enum.to_list(-30..-1) - assert Enum.count(plot) == 30 - assert Enum.any?(plot, fn pageviews -> pageviews > 0 end) + assert length(meta["time_labels"]) == 30 + assert List.first(meta["time_labels"]) == "2024-04-02 02:57:00" + assert List.last(meta["time_labels"]) == "2024-04-02 03:26:00" + assert [%{"dimensions" => ["2024-04-02 03:20:00"], "metrics" => [1]}] = results end test "displays pageviews for the last 30 minutes for a non-UTC timezone site", %{ @@ -28,16 +54,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do |> Plausible.Repo.update() populate_stats(site, [ - build(:pageview, timestamp: relative_time(minute: -5)) + build(:pageview, timestamp: ~N[2024-04-02 03:20:46]) ]) - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime&metric=pageviews") + response = + do_query( + conn, + site, + %{ + "date_range" => "realtime_30m", + "metrics" => ["pageviews"], + "dimensions" => ["time:minute"], + "include" => %{"time_labels" => true} + }, + now: ~U[2024-04-02 03:27:30Z] + ) - assert %{"plot" => plot, "labels" => labels} = json_response(conn, 200) + %{"results" => results, "meta" => meta} = response - assert labels == Enum.to_list(-30..-1) - assert Enum.count(plot) == 30 - assert Enum.any?(plot, fn pageviews -> pageviews > 0 end) + assert length(meta["time_labels"]) == 30 + assert List.first(meta["time_labels"]) == "2024-04-02 05:57:00" + assert List.last(meta["time_labels"]) == "2024-04-02 06:26:00" + assert [%{"dimensions" => ["2024-04-02 06:20:00"], "metrics" => [1]}] = results end test "displays pageviews for a day", %{conn: conn, site: site} do @@ -46,18 +84,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-01 23:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=pageviews" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - zeroes = List.duplicate(0, 22) - - assert Enum.count(plot) == 24 - assert plot == [1] ++ zeroes ++ [1] + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["pageviews"], + "dimensions" => ["time:hour"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01 23:00:00"], "metrics" => [1]} + ] end test "returns empty plot with no native data and recently imported from ga in realtime graph", @@ -67,14 +105,16 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: Date.utc_today()) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=realtime&with_imported=true" - ) + response = + do_query(conn, site, %{ + "date_range" => "realtime_30m", + "metrics" => ["visitors"], + "dimensions" => ["time:minute"], + "include" => %{"imports" => true, "time_labels" => true} + }) - zeroes = List.duplicate(0, 30) - assert %{"plot" => ^zeroes} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 30 + assert response["results"] == [] end test "imported data is not included for hourly interval", %{ @@ -88,15 +128,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [1] ++ List.duplicate(0, 23) + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1]} + ] end test "displays hourly stats in configured timezone", %{conn: conn, user: user} do @@ -112,18 +155,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - zeroes = List.duplicate(0, 22) + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"] + }) # Expecting pageview to show at 1am CET - assert plot == [0, 1] ++ zeroes + assert response["results"] == [ + %{"dimensions" => ["2021-01-01 01:00:00"], "metrics" => [1]} + ] end test "displays visitors for a month", %{conn: conn, site: site} do @@ -132,18 +175,21 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "include" => %{"time_labels" => true}, + "dimensions" => ["time:day"] + }) - assert %{"plot" => plot} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 31 - assert Enum.count(plot) == 31 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [1]} + ] end test "displays visitors for last 28d", %{conn: conn, site: site} do @@ -152,18 +198,21 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-28 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=28d&date=2021-01-29&metric=visitors" - ) + response = + do_query(conn, site, %{ + "date_range" => "28d", + "relative_date" => "2021-01-29", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true} + }) - assert %{"plot" => plot} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 28 - assert Enum.count(plot) == 28 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-28"], "metrics" => [1]} + ] end test "displays visitors for last 91d", %{conn: conn, site: site} do @@ -172,18 +221,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-04-16 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=91d&date=2021-04-17&metric=visitors" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 91 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + response = + do_query(conn, site, %{ + "date_range" => "91d", + "relative_date" => "2021-04-17", + "metrics" => ["visitors"], + "dimensions" => ["time:day"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-16"], "metrics" => [1]}, + %{"dimensions" => ["2021-04-16"], "metrics" => [1]} + ] end test "displays visitors for a month with imported data", %{conn: conn, site: site} do @@ -194,18 +243,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [2]} + ] end test "displays visitors for a month with only imported data", %{conn: conn, site: site} do @@ -214,18 +264,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [1]} + ] end test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} do @@ -236,20 +287,20 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - filters = Jason.encode!([[:is, "event:page", ["/pageA"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:page", ["/pageA"]]], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [1]} + ] end test "displays visitors for 6 months with imported data", %{conn: conn, site: site} do @@ -260,18 +311,30 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-05-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true" - ) + response = + do_query(conn, site, %{ + "date_range" => "6mo", + "relative_date" => "2021-06-30", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true, "time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2020-12-01", + "2021-01-01", + "2021-02-01", + "2021-03-01", + "2021-04-01", + "2021-05-01" + ] - assert %{"plot" => plot} = json_response(conn, 200) + assert response["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-05-01"], "metrics" => [2]} + ] - assert Enum.count(plot) == 6 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + assert response["meta"]["time_label_result_indices"] == [0, nil, nil, nil, nil, 1] end test "displays visitors for 6 months with only imported data", %{conn: conn, site: site} do @@ -280,18 +343,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-05-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 6 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + response = + do_query(conn, site, %{ + "date_range" => "6mo", + "relative_date" => "2021-06-30", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-05-01"], "metrics" => [1]} + ] end test "displays visitors for 12 months with imported data", %{conn: conn, site: site} do @@ -302,18 +366,21 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-11-30]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true" - ) + response = + do_query(conn, site, %{ + "date_range" => "12mo", + "relative_date" => "2021-12-31", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true, "time_labels" => true} + }) - assert %{"plot" => plot} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 12 - assert Enum.count(plot) == 12 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + assert response["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-11-01"], "metrics" => [2]} + ] end test "displays visitors for 12 months with only imported data", %{conn: conn, site: site} do @@ -322,18 +389,37 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-11-30]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true" - ) + response = + do_query(conn, site, %{ + "date_range" => "12mo", + "relative_date" => "2021-12-31", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true, "time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2020-12-01", + "2021-01-01", + "2021-02-01", + "2021-03-01", + "2021-04-01", + "2021-05-01", + "2021-06-01", + "2021-07-01", + "2021-08-01", + "2021-09-01", + "2021-10-01", + "2021-11-01" + ] - assert %{"plot" => plot} = json_response(conn, 200) + assert response["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-11-01"], "metrics" => [1]} + ] - assert Enum.count(plot) == 12 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + assert response["meta"]["time_label_result_indices"] == + [0] ++ List.duplicate(nil, 10) ++ [1] end test "displays visitors for calendar year with imported data", %{conn: conn, site: site} do @@ -344,18 +430,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-12-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 12 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + response = + do_query(conn, site, %{ + "date_range" => "year", + "relative_date" => "2021-12-31", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-12-01"], "metrics" => [2]} + ] end test "displays visitors for calendar year with only imported data", %{conn: conn, site: site} do @@ -364,18 +451,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-12-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 12 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + response = + do_query(conn, site, %{ + "date_range" => "year", + "relative_date" => "2021-12-31", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-12-01"], "metrics" => [1]} + ] end test "displays visitors for all time with just native data", %{conn: conn, site: site} do @@ -391,25 +479,43 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-12-31 00:00:00]) ]) - conn = - get( + response = + do_query( conn, - "/api/stats/#{site.domain}/main-graph?period=all&with_imported=true" + site, + %{ + "date_range" => "all", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true, "time_labels" => true} + }, + now: ~U[2022-03-15 10:00:00Z] ) - assert %{"plot" => plot} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 27 + assert List.last(response["meta"]["time_labels"]) == "2022-03-01" - assert List.first(plot) == 1 - assert Enum.sum(plot) == 3 + assert response["results"] == [ + %{"dimensions" => ["2020-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-12-01"], "metrics" => [1]} + ] end end - describe "GET /api/stats/main-graph - default labels" do + describe "default labels" do setup [:create_user, :log_in, :create_site] test "shows last 30 days", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=30d&metric=visitors") - assert %{"labels" => labels} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => "30d", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true} + }) + + labels = response["meta"]["time_labels"] first = Date.utc_today() |> Date.shift(day: -30) |> Date.to_iso8601() last = Date.utc_today() |> Date.shift(day: -1) |> Date.to_iso8601() @@ -418,8 +524,15 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end test "shows last 7 days", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=7d&metric=visitors") - assert %{"labels" => labels} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => "7d", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true} + }) + + labels = response["meta"]["time_labels"] first = Date.utc_today() |> Date.shift(day: -7) |> Date.to_iso8601() last = Date.utc_today() |> Date.shift(day: -1) |> Date.to_iso8601() @@ -428,7 +541,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end end - describe "GET /api/stats/main-graph - pageviews plot" do + describe "pageviews plot" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "displays pageviews for a month", %{conn: conn, site: site} do @@ -438,17 +551,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 2 - assert List.last(plot) == 1 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["pageviews"], + "dimensions" => ["time:day"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [1]} + ] end test "displays pageviews for a month with imported data", %{conn: conn, site: site} do @@ -459,18 +573,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 2 - assert List.last(plot) == 2 - assert Enum.sum(plot) == 4 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["pageviews"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [2]} + ] end test "displays pageviews for a month with only imported data", %{conn: conn, site: site} do @@ -479,22 +594,84 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["pageviews"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [1]} + ] + end + end - assert %{"plot" => plot} = json_response(conn, 200) + describe "views_per_visit plot" do + setup [:create_user, :log_in, :create_site, :create_legacy_site_import] - assert Enum.count(plot) == 31 - assert List.first(plot) == 1 - assert List.last(plot) == 1 - assert Enum.sum(plot) == 2 + test "views_per_visit for 28 days in weekly buckets (native data only)", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, user_id: 1, timestamp: ~N[2021-01-04 00:00:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-01-04 00:05:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-18 00:00:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-18 00:05:00]), + build(:pageview, user_id: 2, timestamp: ~N[2021-01-18 00:10:00]) + ]) + + response = + do_query(conn, site, %{ + "date_range" => "28d", + "relative_date" => "2021-01-29", + "metrics" => ["views_per_visit"], + "dimensions" => ["time:week"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-04"], "metrics" => [2.0]}, + %{"dimensions" => ["2021-01-18"], "metrics" => [3.0]} + ] + end + + test "views_per_visit for a year in monthly buckets (with imported data)", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # January 2021 - only imported + build(:imported_visitors, date: ~D[2021-01-01], visits: 6, pageviews: 7), + # March 2021 - imported + native combined + build(:imported_visitors, date: ~D[2021-03-01], visits: 1, pageviews: 4), + build(:pageview, user_id: 1, timestamp: ~N[2021-03-15 00:00:00]), + build(:pageview, user_id: 1, timestamp: ~N[2021-03-15 00:05:00]), + # September 2021 - only native + build(:pageview, user_id: 2, timestamp: ~N[2021-09-01 00:00:00]) + ]) + + response = + do_query(conn, site, %{ + "date_range" => "year", + "relative_date" => "2021-01-01", + "metrics" => ["views_per_visit"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1.17]}, + %{"dimensions" => ["2021-03-01"], "metrics" => [3.0]}, + %{"dimensions" => ["2021-09-01"], "metrics" => [1.0]} + ] end end - describe "GET /api/stats/main-graph - visitors plot" do + describe "visitors plot" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "displays visitors per hour with short visits", %{ @@ -507,17 +684,17 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-01 00:20:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=hour" - ) - - assert %{"plot" => plot} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"] + }) - assert Enum.count(plot) == 24 - assert List.first(plot) == 2 - assert Enum.sum(plot) == 2 + assert response["results"] == [ + %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [2]} + ] end test "displays visitors realtime with visits spanning multiple minutes", %{ @@ -525,24 +702,47 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do site: site } do populate_stats(site, [ - build(:pageview, timestamp: relative_time(minute: -35), user_id: 1), - build(:pageview, timestamp: relative_time(minute: -20), user_id: 1), - build(:pageview, timestamp: relative_time(minute: -25), user_id: 2), - build(:pageview, timestamp: relative_time(minute: -15), user_id: 2), - build(:pageview, timestamp: relative_time(minute: -5), user_id: 3), - build(:pageview, timestamp: relative_time(minute: -3), user_id: 3) + build(:pageview, timestamp: ~N[2023-09-10 15:15:00], user_id: 1), + build(:pageview, timestamp: ~N[2023-09-10 15:30:00], user_id: 1), + build(:pageview, timestamp: ~N[2023-09-10 15:25:00], user_id: 2), + build(:pageview, timestamp: ~N[2023-09-10 15:35:00], user_id: 2), + build(:pageview, timestamp: ~N[2023-09-10 15:45:00], user_id: 3), + build(:pageview, timestamp: ~N[2023-09-10 15:47:00], user_id: 3) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=realtime&metric=visitors" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - expected_plot = ~w[1 1 1 1 1 2 2 2 2 2 2 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 0 0] - assert plot == Enum.map(expected_plot, &String.to_integer/1) + response = + do_query( + conn, + site, + %{ + "date_range" => "realtime_30m", + "metrics" => ["visitors"], + "dimensions" => ["time:minute"] + }, + now: ~U[2023-09-10 15:50:01Z] + ) + + assert response["results"] == [ + %{"dimensions" => ["2023-09-10 15:20:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:21:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:22:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:23:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:24:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:25:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:26:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:27:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:28:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:29:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:30:00"], "metrics" => [2]}, + %{"dimensions" => ["2023-09-10 15:31:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:32:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:33:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:34:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:35:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:45:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:46:00"], "metrics" => [1]}, + %{"dimensions" => ["2023-09-10 15:47:00"], "metrics" => [1]} + ] end test "displays visitors per hour with visits spanning multiple hours", %{ @@ -562,83 +762,101 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-02 00:05:00], user_id: 3) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=hour" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - zeroes = List.duplicate(0, 20) - assert [2, 1, 1] ++ zeroes ++ [1] == plot + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-01 01:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01 02:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01 23:00:00"], "metrics" => [1]} + ] end - test "displays visitors per day with visits showed only in last time bucket", %{ - conn: conn, - site: site - } do + test "displays visitors per day with sessions being counted only in the last time bucket they were active in", + %{ + conn: conn, + site: site + } do populate_stats(site, [ build(:pageview, timestamp: ~N[2020-12-31 23:45:00], user_id: 1), build(:pageview, timestamp: ~N[2021-01-01 00:10:00], user_id: 1), - build(:pageview, timestamp: ~N[2020-01-02 23:45:00], user_id: 2), + build(:pageview, timestamp: ~N[2021-01-02 23:45:00], user_id: 2), build(:pageview, timestamp: ~N[2021-01-03 00:10:00], user_id: 2), - build(:pageview, timestamp: ~N[2020-01-03 23:45:00], user_id: 3), + build(:pageview, timestamp: ~N[2021-01-03 23:45:00], user_id: 3), build(:pageview, timestamp: ~N[2021-01-04 00:10:00], user_id: 3), - build(:pageview, timestamp: ~N[2020-01-07 23:45:00], user_id: 4), + build(:pageview, timestamp: ~N[2021-01-07 23:45:00], user_id: 4), build(:pageview, timestamp: ~N[2021-01-08 00:10:00], user_id: 4) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2021-01-08&metric=visitors&interval=day" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - assert plot == [1, 0, 1, 1, 0, 0, 0] + response = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2021-01-08", + "metrics" => ["visitors"], + "dimensions" => ["time:day"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-04"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-07"], "metrics" => [1]} + ] end - test "displays visitors per week with visits showed only in last time bucket", %{ - conn: conn, - site: site - } do + test "displays visitors per week with sessions being counted only in the last time bucket they were active in", + %{ + conn: conn, + site: site + } do populate_stats(site, [ build(:pageview, timestamp: ~N[2020-12-31 23:45:00], user_id: 1), build(:pageview, timestamp: ~N[2021-01-01 00:10:00], user_id: 1), - build(:pageview, timestamp: ~N[2020-01-03 23:45:00], user_id: 2), + build(:pageview, timestamp: ~N[2021-01-03 23:45:00], user_id: 2), build(:pageview, timestamp: ~N[2021-01-04 00:10:00], user_id: 2), build(:pageview, timestamp: ~N[2021-01-31 23:45:00], user_id: 3), build(:pageview, timestamp: ~N[2021-02-01 00:05:00], user_id: 3) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=week" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [1, 1, 0, 0, 0] + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:week"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-04"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-25"], "metrics" => [1]} + ] end end - describe "GET /api/stats/main-graph - scroll_depth plot" do + describe "scroll_depth plot" do setup [:create_user, :log_in, :create_site] test "returns 400 when scroll_depth is queried without a page filter", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=scroll_depth" - ) + response = + do_query_fail(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["scroll_depth"], + "dimensions" => ["time:day"] + }) - assert %{"error" => error} = json_response(conn, 400) - assert error =~ "can only be queried with a page filter" + assert %{"error" => error} = json_response(response, 400) + assert error =~ "can only be queried with event:page filters or dimensions" end test "returns scroll depth per day", %{conn: conn, site: site} do @@ -660,17 +878,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - filters = Jason.encode!([[:is, "event:page", ["/"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2020-01-08&metric=scroll_depth&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [40, 20, nil, nil, nil, nil, nil] + response = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2020-01-08", + "metrics" => ["scroll_depth"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:page", ["/"]]] + }) + + assert response["results"] == [ + %{"dimensions" => ["2020-01-01"], "metrics" => [40]}, + %{"dimensions" => ["2020-01-02"], "metrics" => [20]} + ] end test "returns scroll depth per day with imported data", %{conn: conn, site: site} do @@ -705,35 +925,41 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100) ]) - filters = Jason.encode!([[:is, "event:page", ["/"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2020-01-08&metric=scroll_depth&filters=#{filters}&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [40, 30, 90, nil, nil, nil, nil] + response = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2020-01-08", + "metrics" => ["scroll_depth"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:page", ["/"]]], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2020-01-01"], "metrics" => [40]}, + %{"dimensions" => ["2020-01-02"], "metrics" => [30]}, + %{"dimensions" => ["2020-01-03"], "metrics" => [90]} + ] end end - describe "GET /api/stats/main-graph - conversion_rate plot" do + describe "conversion_rate plot" do setup [:create_user, :log_in, :create_site] test "returns 400 when conversion rate is queried without a goal filter", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=conversion_rate" - ) + response = + do_query_fail(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["conversion_rate"], + "dimensions" => ["time:day"] + }) - assert %{"error" => error} = json_response(conn, 400) - assert error =~ "can only be queried with a goal filter" + assert %{"error" => error} = json_response(response, 400) + assert error =~ "can only be queried with event:goal filters or dimensions" end test "displays conversion_rate for a month", %{conn: conn, site: site} do @@ -747,40 +973,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:event, name: "Signup", timestamp: ~N[2021-01-31 00:00:00]) ]) - filters = Jason.encode!([[:is, "event:goal", ["Signup"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=conversion_rate&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - assert Enum.count(plot) == 31 - - assert List.first(plot) == 33.33 - assert Enum.at(plot, 10) == 0.0 - assert List.last(plot) == 50.0 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["group_conversion_rate"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["Signup"]]] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [33.33]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [50.0]} + ] end end - describe "GET /api/stats/main-graph - events (total conversions) plot" do + describe "events (total conversions) plot" do setup [:create_user, :log_in, :create_site] - test "returns 400 when the `events` metric is queried without a goal filter", %{ - conn: conn, - site: site - } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=events" - ) - - assert %{"error" => error} = json_response(conn, 400) - assert error =~ "`events` can only be queried with a goal filter" - end - test "displays total conversions for a goal", %{conn: conn, site: site} do insert(:goal, site: site, event_name: "Signup") @@ -794,20 +1005,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:event, name: "Signup", user_id: 123, timestamp: ~N[2021-01-31 00:00:00]) ]) - filters = Jason.encode!([[:is, "event:goal", ["Signup"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=events&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - assert Enum.count(plot) == 31 - - assert List.first(plot) == 2 - assert Enum.at(plot, 10) == 0.0 - assert List.last(plot) == 3 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["events"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["Signup"]]] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [3]} + ] end test "displays total conversions per hour with previous day comparison plot", %{ @@ -827,17 +1037,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:event, name: "Signup", timestamp: ~N[2021-01-11 18:00:00]) ]) - filters = Jason.encode!([[:is, "event:goal", ["Signup"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-11&metric=events&filters=#{filters}&comparison=previous_period" - ) + %{"results" => results, "comparison_results" => comparison_results} = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-11", + "metrics" => ["events"], + "dimensions" => ["time:hour"], + "filters" => [["is", "event:goal", ["Signup"]]], + "include" => %{"compare" => "previous_period"} + }) + + assert results == [ + %{"dimensions" => ["2021-01-11 04:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-11 05:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-11 18:00:00"], "metrics" => [1]} + ] - assert %{"plot" => curr, "comparison_plot" => prev} = json_response(conn, 200) - assert [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] = prev - assert [0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] = curr + assert comparison_results == [ + %{"dimensions" => ["2021-01-10 05:00:00"], "metrics" => [2], "change" => [-50]}, + %{"dimensions" => ["2021-01-10 19:00:00"], "metrics" => [1], "change" => nil} + ] end test "displays conversions per month with 12mo comparison plot", %{ @@ -857,41 +1076,52 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:event, name: "Signup", timestamp: ~N[2021-07-11 00:00:00]) ]) - filters = Jason.encode!([[:is, "event:goal", ["Signup"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-11&metric=events&filters=#{filters}&comparison=previous_period" - ) + %{"results" => results, "comparison_results" => comparison_results} = + do_query(conn, site, %{ + "date_range" => "12mo", + "relative_date" => "2021-12-11", + "metrics" => ["events"], + "dimensions" => ["time:month"], + "filters" => [["is", "event:goal", ["Signup"]]], + "include" => %{"compare" => "previous_period"} + }) + + assert results == [ + %{"dimensions" => ["2021-05-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-06-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-07-01"], "metrics" => [1]} + ] - assert %{"plot" => curr, "comparison_plot" => prev} = json_response(conn, 200) - assert [0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] = prev - assert [0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0] = curr + assert comparison_results == [ + %{"dimensions" => ["2020-01-01"], "metrics" => [1], "change" => nil}, + %{"dimensions" => ["2020-02-01"], "metrics" => [1], "change" => nil}, + %{"dimensions" => ["2020-03-01"], "metrics" => [1], "change" => nil} + ] end end - describe "GET /api/stats/main-graph - bounce_rate plot" do + describe "bounce_rate plot" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "displays bounce_rate for a month", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), - build(:pageview, timestamp: ~N[2021-01-03 00:10:00]), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00], user_id: 1), + build(:pageview, timestamp: ~N[2021-01-03 00:10:00], user_id: 1), build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 0 - assert List.last(plot) == 100 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["bounce_rate"], + "dimensions" => ["time:day"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-03"], "metrics" => [0]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [100]} + ] end test "displays bounce rate for a month with imported data", %{conn: conn, site: site} do @@ -902,17 +1132,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 50 - assert List.last(plot) == 100 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["bounce_rate"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [50]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [100]} + ] end test "displays bounce rate for a month with only imported data", %{conn: conn, site: site} do @@ -921,21 +1153,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 0 - assert List.last(plot) == 100 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["bounce_rate"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [0]}, + %{"dimensions" => ["2021-01-31"], "metrics" => [100]} + ] end end - describe "GET /api/stats/main-graph - visit_duration plot" do + describe "visit_duration plot" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "displays visit_duration for a month", %{conn: conn, site: site} do @@ -952,17 +1186,17 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration" - ) - - assert %{"plot" => plot} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visit_duration"], + "dimensions" => ["time:day"] + }) - assert Enum.count(plot) == 31 - assert List.first(plot) == nil - assert List.last(plot) == 300 + assert response["results"] == [ + %{"dimensions" => ["2021-01-31"], "metrics" => [300]} + ] end test "displays visit_duration for a month with imported data", %{conn: conn, site: site} do @@ -972,16 +1206,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 200 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visit_duration"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [200]} + ] end test "displays visit_duration for a month with only imported data", %{conn: conn, site: site} do @@ -989,20 +1225,22 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 31 - assert List.first(plot) == 100 + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visit_duration"], + "dimensions" => ["time:day"], + "include" => %{"imports" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [100]} + ] end end - describe "GET /api/stats/main-graph - varying intervals" do + describe "varying intervals" do setup [:create_user, :log_in, :create_site] test "displays visitors for 6mo on a day scale", %{conn: conn, site: site} do @@ -1014,19 +1252,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-05-31 01:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-01&metric=visitors&interval=day" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 182 - assert List.first(plot) == 1 - assert Enum.at(plot, 14) == 2 - assert Enum.at(plot, 45) == 1 - assert List.last(plot) == 1 + response = + do_query(conn, site, %{ + "date_range" => "6mo", + "relative_date" => "2021-06-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true} + }) + + assert length(response["meta"]["time_labels"]) == 182 + + assert response["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-15"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-15"], "metrics" => [1]}, + %{"dimensions" => ["2021-05-31"], "metrics" => [1]} + ] end test "displays visitors for a custom period on a monthly scale", %{conn: conn, site: site} do @@ -1037,50 +1279,34 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-06-01 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-06-30&metric=visitors&interval=month" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert Enum.count(plot) == 6 - assert List.first(plot) == 2 - assert Enum.at(plot, 1) == 1 - assert List.last(plot) == 1 - end - - test "returns error when requesting an interval longer than the time period", %{ - conn: conn, - site: site - } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=month" - ) - - assert %{ - "error" => - "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`" - } == json_response(conn, 400) + response = + do_query(conn, site, %{ + "date_range" => ["2021-01-01", "2021-06-30"], + "metrics" => ["visitors"], + "dimensions" => ["time:month"] + }) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-02-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-06-01"], "metrics" => [1]} + ] end test "returns error when the interval is not valid", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=biweekly" - ) + response = + do_query_fail(conn, site, %{ + "date_range" => "day", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:biweekly"] + }) - assert %{ - "error" => - "Invalid value for interval. Accepted values are: minute, hour, day, week, month" - } == json_response(conn, 400) + assert %{"error" => error} = json_response(response, 400) + assert error =~ "Invalid dimensions" end test "displays visitors for a month on a weekly scale", %{conn: conn, site: site} do @@ -1090,40 +1316,45 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-05 00:15:02]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=week" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true} + }) - assert %{"plot" => plot} = json_response(conn, 200) + assert length(response["meta"]["time_labels"]) == 5 - assert Enum.count(plot) == 5 - assert List.first(plot) == 2 - assert Enum.at(plot, 1) == 1 + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-04"], "metrics" => [1]} + ] end - test "shows imperfect week-split month on week scale with full week indicators", %{ + test "shows imperfect week-split month on week scale with partial week indicators", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-09-01" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-09-01", "2021-09-06", "2021-09-13", "2021-09-20", "2021-09-27"] + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-09-01", + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2021-09-01", + "2021-09-06", + "2021-09-13", + "2021-09-20", + "2021-09-27" + ] - assert full_intervals == %{ - "2021-09-01" => false, - "2021-09-06" => true, - "2021-09-13" => true, - "2021-09-20" => true, - "2021-09-27" => false - } + assert response["meta"]["partial_time_labels"] == ["2021-09-01", "2021-09-27"] end test "returns stats for the first week of the month when site timezone is ahead of UTC", %{ @@ -1139,39 +1370,44 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2023-03-01 12:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&date=2023-03-01&interval=week" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2023-03-01", + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true} + }) - %{"labels" => labels, "plot" => plot} = json_response(conn, 200) + assert List.first(response["meta"]["time_labels"]) == "2023-03-01" - assert List.first(plot) == 1 - assert List.first(labels) == "2023-03-01" + assert response["results"] == [ + %{"metrics" => [1], "dimensions" => ["2023-03-01"]} + ] end - test "shows half-perfect week-split month on week scale with full week indicators", %{ + test "shows half-perfect week-split month on week scale with partial week indicators", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-10-01" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-10-01", "2021-10-04", "2021-10-11", "2021-10-18", "2021-10-25"] + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-10-01", + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2021-10-01", + "2021-10-04", + "2021-10-11", + "2021-10-18", + "2021-10-25" + ] - assert full_intervals == %{ - "2021-10-01" => false, - "2021-10-04" => true, - "2021-10-11" => true, - "2021-10-18" => true, - "2021-10-25" => true - } + assert response["meta"]["partial_time_labels"] == ["2021-10-01"] end test "shows perfect week-split range on week scale with full week indicators for custom period", @@ -1179,15 +1415,15 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=week&from=2020-12-21&to=2021-02-07" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == [ + response = + do_query(conn, site, %{ + "date_range" => ["2020-12-21", "2021-02-07"], + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ "2020-12-21", "2020-12-28", "2021-01-04", @@ -1197,125 +1433,71 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "2021-02-01" ] - assert full_intervals == %{ - "2020-12-21" => true, - "2020-12-28" => true, - "2021-01-04" => true, - "2021-01-11" => true, - "2021-01-18" => true, - "2021-01-25" => true, - "2021-02-01" => true - } + assert response["meta"]["partial_time_labels"] == [] end test "shows imperfect week-split for last 28d with full week indicators", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=28d&metric=visitors&interval=week&date=2021-10-30" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-10-02", "2021-10-04", "2021-10-11", "2021-10-18", "2021-10-25"] - - assert full_intervals == %{ - "2021-10-02" => false, - "2021-10-04" => true, - "2021-10-11" => true, - "2021-10-18" => true, - "2021-10-25" => false - } - end - - test "shows perfect week-split for last 28d with full week indicators", %{ - conn: conn, - site: site - } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=28d&date=2021-02-08&metric=visitors&interval=week" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-01-11", "2021-01-18", "2021-01-25", "2021-02-01"] + response = + do_query(conn, site, %{ + "date_range" => "28d", + "relative_date" => "2021-10-30", + "metrics" => ["visitors"], + "dimensions" => ["time:week"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2021-10-02", + "2021-10-04", + "2021-10-11", + "2021-10-18", + "2021-10-25" + ] - assert full_intervals == %{ - "2021-01-11" => true, - "2021-01-18" => true, - "2021-01-25" => true, - "2021-02-01" => true - } + assert response["meta"]["partial_time_labels"] == ["2021-10-02", "2021-10-25"] end test "shows imperfect month-split for custom period with full month indicators", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=month&from=2021-09-06&to=2021-12-13" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"] - - assert full_intervals == %{ - "2021-09-01" => false, - "2021-10-01" => true, - "2021-11-01" => true, - "2021-12-01" => false - } - end - - test "shows imperfect month-split for last 91d with full month indicators", %{ - conn: conn, - site: site - } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=91d&metric=visitors&interval=month&date=2021-12-13" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) - - assert labels == ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"] + response = + do_query(conn, site, %{ + "date_range" => ["2021-09-06", "2021-12-13"], + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) + + assert response["meta"]["time_labels"] == [ + "2021-09-01", + "2021-10-01", + "2021-11-01", + "2021-12-01" + ] - assert full_intervals == %{ - "2021-09-01" => false, - "2021-10-01" => true, - "2021-11-01" => true, - "2021-12-01" => false - } + assert response["meta"]["partial_time_labels"] == ["2021-09-01", "2021-12-01"] end test "shows perfect month-split for last 91d with full month indicators", %{ conn: conn, site: site } do - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=91d&metric=visitors&interval=month&date=2021-12-01" - ) - - assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => "91d", + "relative_date" => "2021-12-01", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"time_labels" => true, "partial_time_labels" => true} + }) - assert labels == ["2021-09-01", "2021-10-01", "2021-11-01"] + assert response["meta"]["time_labels"] == ["2021-09-01", "2021-10-01", "2021-11-01"] - assert full_intervals == %{ - "2021-09-01" => true, - "2021-10-01" => true, - "2021-11-01" => true - } + assert response["meta"]["partial_time_labels"] == [] end test "returns stats for a day with a minute interval", %{conn: conn, site: site} do @@ -1323,21 +1505,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2023-03-01 12:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2023-03-01&interval=minute" - ) + response = + do_query(conn, site, %{ + "date_range" => "day", + "relative_date" => "2023-03-01", + "metrics" => ["visitors"], + "dimensions" => ["time:minute"], + "include" => %{"time_labels" => true} + }) - %{"labels" => labels, "plot" => plot} = json_response(conn, 200) + labels = response["meta"]["time_labels"] assert length(labels) == 24 * 60 - assert List.first(labels) == "2023-03-01 00:00:00" assert Enum.at(labels, 1) == "2023-03-01 00:01:00" assert List.last(labels) == "2023-03-01 23:59:00" - assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1 + assert response["results"] == [ + %{"dimensions" => ["2023-03-01 12:00:00"], "metrics" => [1]} + ] end test "trims hourly relative date range", %{conn: conn, site: site} do @@ -1348,27 +1534,37 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-08 23:59:00]) ]) - conn = - conn - |> Plug.Conn.put_private(:now, ~U[2021-01-08 08:05:00Z]) - |> get( - "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour" - ) + response = + do_query( + conn, + site, + %{ + "date_range" => "day", + "relative_date" => "2021-01-08", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"], + "include" => %{"time_labels" => true} + }, + now: ~U[2021-01-08 08:05:00Z] + ) + + assert response["meta"]["time_labels"] == [ + "2021-01-08 00:00:00", + "2021-01-08 01:00:00", + "2021-01-08 02:00:00", + "2021-01-08 03:00:00", + "2021-01-08 04:00:00", + "2021-01-08 05:00:00", + "2021-01-08 06:00:00", + "2021-01-08 07:00:00", + "2021-01-08 08:00:00" + ] - assert_matches %{ - "labels" => [ - "2021-01-08 00:00:00", - "2021-01-08 01:00:00", - "2021-01-08 02:00:00", - "2021-01-08 03:00:00", - "2021-01-08 04:00:00", - "2021-01-08 05:00:00", - "2021-01-08 06:00:00", - "2021-01-08 07:00:00", - "2021-01-08 08:00:00" - ], - "plot" => [1, 0, 0, 0, 0, 0, 1, 0, 1] - } = json_response(conn, 200) + assert response["results"] == [ + %{"dimensions" => ["2021-01-08 00:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-08 06:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-08 08:00:00"], "metrics" => [1]} + ] end test "trims monthly relative date range", %{conn: conn, site: site} do @@ -1379,25 +1575,35 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) ]) - conn = - conn - |> Plug.Conn.put_private(:now, ~U[2021-01-07 12:00:00Z]) - |> get( - "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&date=2021-01-07&interval=day" + response = + do_query( + conn, + site, + %{ + "date_range" => "month", + "relative_date" => "2021-01-07", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true} + }, + now: ~U[2021-01-07 12:00:00Z] ) - assert_matches %{ - "labels" => [ - "2021-01-01", - "2021-01-02", - "2021-01-03", - "2021-01-04", - "2021-01-05", - "2021-01-06", - "2021-01-07" - ], - "plot" => [1, 0, 0, 0, 1, 0, 1] - } = json_response(conn, 200) + assert response["meta"]["time_labels"] == [ + "2021-01-01", + "2021-01-02", + "2021-01-03", + "2021-01-04", + "2021-01-05", + "2021-01-06", + "2021-01-07" + ] + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-05"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-07"], "metrics" => [1]} + ] end test "trims yearly relative date range", %{conn: conn, site: site} do @@ -1410,35 +1616,46 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-02-09 00:00:00]) ]) - conn = - conn - |> Plug.Conn.put_private(:now, ~U[2021-02-07 12:00:00Z]) - |> get( - "/api/stats/#{site.domain}/main-graph?period=year&metric=visitors&date=2021-02-07&interval=month" + response = + do_query( + conn, + site, + %{ + "date_range" => "year", + "relative_date" => "2021-02-07", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"time_labels" => true} + }, + now: ~U[2021-02-07 12:00:00Z] ) - assert_matches %{ - "labels" => [ - "2021-01-01", - "2021-02-01" - ], - "plot" => [4, 1] - } = json_response(conn, 200) + assert response["meta"]["time_labels"] == ["2021-01-01", "2021-02-01"] + + assert response["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [4]}, + %{"dimensions" => ["2021-02-01"], "metrics" => [1]} + ] end end - describe "GET /api/stats/main-graph - comparisons" do + describe "comparisons" do setup [:create_user, :log_in, :create_site, :create_legacy_site_import] test "returns past month stats when period=30d and comparison=previous_period", %{ conn: conn, site: site } do - conn = - get(conn, "/api/stats/#{site.domain}/main-graph?period=30d&comparison=previous_period") + response = + do_query(conn, site, %{ + "date_range" => "30d", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"compare" => "previous_period", "time_labels" => true} + }) - assert %{"labels" => labels, "comparison_labels" => comparison_labels} = - json_response(conn, 200) + labels = response["meta"]["time_labels"] + comparison_labels = response["meta"]["comparison_time_labels"] first = Date.utc_today() |> Date.shift(day: -30) |> Date.to_iso8601() last = Date.utc_today() |> Date.shift(day: -1) |> Date.to_iso8601() @@ -1469,25 +1686,39 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2019-01-31 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2020-01-01&comparison=year_over_year" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2020-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{"compare" => "year_over_year", "time_labels" => true} + }) + + assert response["results"] == [ + %{"dimensions" => ["2020-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-01-05"], "metrics" => [1]}, + %{"dimensions" => ["2020-01-30"], "metrics" => [1]}, + %{"dimensions" => ["2020-01-31"], "metrics" => [1]} + ] - assert %{"plot" => plot, "comparison_plot" => comparison_plot} = json_response(conn, 200) + assert response["comparison_results"] == [ + %{"dimensions" => ["2019-01-01"], "metrics" => [2], "change" => [-50]}, + %{"dimensions" => ["2019-01-05"], "metrics" => [2], "change" => [-50]}, + %{"dimensions" => ["2019-01-31"], "metrics" => [1], "change" => [0]} + ] - assert 1 == Enum.at(plot, 0) - assert 2 == Enum.at(comparison_plot, 0) + assert length(response["meta"]["time_labels"]) == 31 + assert length(response["meta"]["comparison_time_labels"]) == 31 - assert 1 == Enum.at(plot, 4) - assert 2 == Enum.at(comparison_plot, 4) + assert response["meta"]["time_label_result_indices"] == + [0, nil, nil, nil, 1] ++ List.duplicate(nil, 24) ++ [2, 3] - assert 1 == Enum.at(plot, 30) - assert 1 == Enum.at(comparison_plot, 30) + assert response["meta"]["comparison_time_label_result_indices"] == + [0, nil, nil, nil, 1] ++ List.duplicate(nil, 25) ++ [2] end - test "fill in gaps when custom comparison period is larger than original query", %{ + test "can return custom comparison period larger than original query", %{ conn: conn, site: site } do @@ -1497,17 +1728,27 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2020-01-30 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2020-01-01&comparison=custom&compare_from=2022-01-01&compare_to=2022-06-01" - ) - - assert %{"labels" => labels, "comparison_plot" => comparison_labels} = - json_response(conn, 200) - - assert length(labels) == length(comparison_labels) - assert "__blank__" == List.last(labels) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2020-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:day"], + "include" => %{ + "compare" => ["2022-01-01", "2022-06-01"], + "time_labels" => true + } + }) + + labels = response["meta"]["time_labels"] + comparison_labels = response["meta"]["comparison_time_labels"] + + assert length(labels) == 31 + assert length(comparison_labels) == 152 + assert length(response["results"]) == 3 + assert response["comparison_results"] == [] + assert List.first(comparison_labels) == "2022-01-01" + assert List.last(comparison_labels) == "2022-06-01" end test "compares imported data and native data together", %{conn: conn, site: site} do @@ -1520,13 +1761,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=year&date=2021-01-01&with_imported=true&comparison=year_over_year&interval=month" - ) + response = + do_query(conn, site, %{ + "date_range" => "year", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => true, "compare" => "year_over_year"} + }) + + plot = Enum.map(response["results"], fn r -> List.first(r["metrics"]) end) - assert %{"plot" => plot, "comparison_plot" => comparison_plot} = json_response(conn, 200) + comparison_plot = + Enum.map(response["comparison_results"], fn r -> List.first(r["metrics"]) end) assert 4 == Enum.sum(plot) assert 2 == Enum.sum(comparison_plot) @@ -1545,13 +1792,19 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=year&date=2021-01-01&with_imported=false&comparison=year_over_year&interval=month" - ) + response = + do_query(conn, site, %{ + "date_range" => "year", + "relative_date" => "2021-01-01", + "metrics" => ["visitors"], + "dimensions" => ["time:month"], + "include" => %{"imports" => false, "compare" => "year_over_year"} + }) + + plot = Enum.map(response["results"], fn r -> List.first(r["metrics"]) end) - assert %{"plot" => plot, "comparison_plot" => comparison_plot} = json_response(conn, 200) + comparison_plot = + Enum.map(response["comparison_results"], fn r -> List.first(r["metrics"]) end) assert 4 == Enum.sum(plot) assert 0 == Enum.sum(comparison_plot) @@ -1568,19 +1821,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-08 00:01:00]) ]) - filters = Jason.encode!([[:is, "event:goal", ["Signup"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2021-01-15&comparison=previous_period&metric=conversion_rate&filters=#{filters}" - ) - - assert %{"plot" => this_week_plot, "comparison_plot" => last_week_plot} = - json_response(conn, 200) + %{"results" => results, "comparison_results" => comparison_results} = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2021-01-15", + "metrics" => ["conversion_rate"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["Signup"]]], + "include" => %{"compare" => "previous_period"} + }) + + assert results == [ + %{"dimensions" => ["2021-01-08"], "metrics" => [50.0]} + ] - assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - assert last_week_plot == [33.33, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + assert comparison_results == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [33.33], "change" => [16.7]} + ] end test "does not trim hourly relative date range when comparing", %{conn: conn, site: site} do @@ -1591,97 +1848,47 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview, timestamp: ~N[2021-01-08 23:59:00]) ]) - conn = - conn - |> Plug.Conn.put_private(:now, ~U[2021-01-08 08:05:00Z]) - |> get( - "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour&comparison=previous_period" - ) + response = + do_query( + conn, + site, + %{ + "date_range" => "day", + "relative_date" => "2021-01-08", + "metrics" => ["visitors"], + "dimensions" => ["time:hour"], + "include" => %{"compare" => "previous_period", "time_labels" => true} + }, + now: ~U[2021-01-08 08:05:00Z] + ) + + assert response["meta"]["time_labels"] == + Enum.map(0..23, fn h -> + "2021-01-08 #{String.pad_leading(to_string(h), 2, "0")}:00:00" + end) + + assert response["results"] == [ + %{"dimensions" => ["2021-01-08 00:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-08 06:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-08 08:00:00"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-08 23:00:00"], "metrics" => [1]} + ] + + assert response["meta"]["comparison_time_labels"] == + Enum.map(0..23, fn h -> + "2021-01-07 #{String.pad_leading(to_string(h), 2, "0")}:00:00" + end) - assert_matches %{ - "labels" => [ - "2021-01-08 00:00:00", - "2021-01-08 01:00:00", - "2021-01-08 02:00:00", - "2021-01-08 03:00:00", - "2021-01-08 04:00:00", - "2021-01-08 05:00:00", - "2021-01-08 06:00:00", - "2021-01-08 07:00:00", - "2021-01-08 08:00:00", - "2021-01-08 09:00:00", - "2021-01-08 10:00:00", - "2021-01-08 11:00:00", - "2021-01-08 12:00:00", - "2021-01-08 13:00:00", - "2021-01-08 14:00:00", - "2021-01-08 15:00:00", - "2021-01-08 16:00:00", - "2021-01-08 17:00:00", - "2021-01-08 18:00:00", - "2021-01-08 19:00:00", - "2021-01-08 20:00:00", - "2021-01-08 21:00:00", - "2021-01-08 22:00:00", - "2021-01-08 23:00:00" - ], - "plot" => [ - 1, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1 - ], - "comparison_plot" => [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ] - } = json_response(conn, 200) + assert response["comparison_results"] == [] + + assert response["meta"]["time_label_result_indices"] == + [0, nil, nil, nil, nil, nil, 1, nil, 2] ++ List.duplicate(nil, 14) ++ [3] + + assert response["meta"]["comparison_time_label_result_indices"] == List.duplicate(nil, 24) end end - describe "GET /api/stats/main-graph - total_revenue plot" do + describe "total_revenue plot" do @describetag :ee_only setup [:create_user, :log_in, :create_site, :create_legacy_site_import] @@ -1715,48 +1922,44 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - filters = Jason.encode!([[:is, "event:goal", ["Payment"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=total_revenue&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [ - %{"currency" => "USD", "long" => "$13.29", "short" => "$13.3", "value" => 13.29}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$30.31", "short" => "$30.3", "value" => 30.31} + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["total_revenue"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["Payment"]]] + }) + + assert response["results"] == [ + %{ + "dimensions" => ["2021-01-01"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$13.29", + "short" => "$13.3", + "value" => 13.29 + } + ] + }, + %{ + "dimensions" => ["2021-01-05"], + "metrics" => [ + %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9} + ] + }, + %{ + "dimensions" => ["2021-01-31"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$30.31", + "short" => "$30.3", + "value" => 30.31 + } + ] + } ] end @@ -1801,43 +2004,65 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - filters = Jason.encode!([[:is, "event:goal", ["PaymentUSD"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2021-01-15&metric=total_revenue&filters=#{filters}&comparison=previous_period" - ) - - assert %{"plot" => plot, "comparison_plot" => prev} = json_response(conn, 200) - - assert plot == [ - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$10.31", "short" => "$10.3", "value" => 10.31}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$30.00", "short" => "$30.0", "value" => 30.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0} + %{"results" => results, "comparison_results" => comparison_results} = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2021-01-15", + "metrics" => ["total_revenue"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["PaymentUSD"]]], + "include" => %{"compare" => "previous_period"} + }) + + assert results == [ + %{ + "dimensions" => ["2021-01-10"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$10.31", + "short" => "$10.3", + "value" => 10.31 + } + ] + }, + %{ + "dimensions" => ["2021-01-12"], + "metrics" => [ + %{"currency" => "USD", "long" => "$30.00", "short" => "$30.0", "value" => 30.0} + ] + } ] - assert prev == [ - %{"currency" => "USD", "long" => "$13.29", "short" => "$13.3", "value" => 13.29}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0} + assert comparison_results == [ + %{ + "dimensions" => ["2021-01-01"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$13.29", + "short" => "$13.3", + "value" => 13.29 + } + ], + "change" => nil + }, + %{ + "dimensions" => ["2021-01-05"], + "metrics" => [ + %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9} + ], + "change" => [51] + } ] end end - describe "GET /api/stats/main-graph - average_revenue plot" do + describe "average_revenue plot" do @describetag :ee_only setup [:create_user, :log_in, :create_site, :create_legacy_site_import] - test "plots total_revenue for a month", %{conn: conn, site: site} do + test "plots average_revenue for a month", %{conn: conn, site: site} do insert(:goal, site: site, event_name: "Payment", currency: "USD") populate_stats(site, [ @@ -1873,48 +2098,44 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - filters = Jason.encode!([[:is, "event:goal", ["Payment"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=average_revenue&filters=#{filters}" - ) - - assert %{"plot" => plot} = json_response(conn, 200) - - assert plot == [ - %{"currency" => "USD", "long" => "$31.90", "short" => "$31.9", "value" => 31.895}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$15.16", "short" => "$15.2", "value" => 15.155} + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["average_revenue"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["Payment"]]] + }) + + assert response["results"] == [ + %{ + "dimensions" => ["2021-01-01"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$31.90", + "short" => "$31.9", + "value" => 31.895 + } + ] + }, + %{ + "dimensions" => ["2021-01-05"], + "metrics" => [ + %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9} + ] + }, + %{ + "dimensions" => ["2021-01-31"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$15.16", + "short" => "$15.2", + "value" => 15.155 + } + ] + } ] end @@ -1959,34 +2180,56 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do ) ]) - filters = Jason.encode!([[:is, "event:goal", ["PaymentUSD"]]]) - - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=7d&date=2021-01-15&metric=average_revenue&filters=#{filters}&comparison=previous_period" - ) - - assert %{"plot" => plot, "comparison_plot" => prev} = json_response(conn, 200) - - assert plot == [ - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$10.31", "short" => "$10.3", "value" => 10.31}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$15.00", "short" => "$15.0", "value" => 15.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0} + %{"results" => results, "comparison_results" => comparison_results} = + do_query(conn, site, %{ + "date_range" => "7d", + "relative_date" => "2021-01-15", + "metrics" => ["average_revenue"], + "dimensions" => ["time:day"], + "filters" => [["is", "event:goal", ["PaymentUSD"]]], + "include" => %{"compare" => "previous_period", "time_labels" => true} + }) + + assert results == [ + %{ + "dimensions" => ["2021-01-10"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$10.31", + "short" => "$10.3", + "value" => 10.31 + } + ] + }, + %{ + "dimensions" => ["2021-01-12"], + "metrics" => [ + %{"currency" => "USD", "long" => "$15.00", "short" => "$15.0", "value" => 15.0} + ] + } ] - assert prev == [ - %{"currency" => "USD", "long" => "$13.29", "short" => "$13.3", "value" => 13.29}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0}, - %{"currency" => "USD", "long" => "$0.00", "short" => "$0.0", "value" => 0.0} + assert comparison_results == [ + %{ + "dimensions" => ["2021-01-01"], + "metrics" => [ + %{ + "currency" => "USD", + "long" => "$13.29", + "short" => "$13.3", + "value" => 13.29 + } + ], + "change" => nil + }, + %{ + "dimensions" => ["2021-01-05"], + "metrics" => [ + %{"currency" => "USD", "long" => "$19.90", "short" => "$19.9", "value" => 19.9} + ], + "change" => [-25] + } ] end end @@ -1999,13 +2242,15 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&metric=pageviews" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "metrics" => ["pageviews"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true, "present_index" => true} + }) - assert %{"present_index" => present_index} = json_response(conn, 200) + present_index = response["meta"]["present_index"] assert present_index >= 0 end @@ -2018,15 +2263,16 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do build(:pageview) ]) - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews" - ) + response = + do_query(conn, site, %{ + "date_range" => "month", + "relative_date" => "2021-01-01", + "metrics" => ["pageviews"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true, "present_index" => true} + }) - assert %{"present_index" => present_index} = json_response(conn, 200) - - refute present_index + refute response["meta"]["present_index"] end for period <- ["7d", "28d", "30d", "91d"] do @@ -2034,13 +2280,17 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do today = "2021-01-01" yesterday = "2020-12-31" - conn = - get( - conn, - "/api/stats/#{site.domain}/main-graph?period=#{unquote(period)}&date=#{today}&metric=pageviews" - ) - - assert %{"labels" => labels, "present_index" => present_index} = json_response(conn, 200) + response = + do_query(conn, site, %{ + "date_range" => unquote(period), + "relative_date" => today, + "metrics" => ["pageviews"], + "dimensions" => ["time:day"], + "include" => %{"time_labels" => true, "present_index" => true} + }) + + labels = response["meta"]["time_labels"] + present_index = response["meta"]["present_index"] refute present_index assert List.last(labels) == yesterday