diff --git a/README.md b/README.md index 20fce78..b184c07 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,16 @@ [![WTFPL](https://img.shields.io/badge/license-WTFPL-brightgreen.svg?style=flat)](https://www.tldrlegal.com/l/wtfpl) [![Last Updated](https://img.shields.io/github/last-commit/werbitzky/elastix.svg)](https://github.com/werbitzky/elastix/commits/master) -A DSL-free Elasticsearch client for Elixir. +An Elasticsearch client for Elixir. + +This library has convenience functions for working with the Elasticsearch +API. It does not use a DSL, it expects you to interact with the Elasticsearch +using native JSON data structures, or the equivalent Elixir data structures which +it will encode for you. + +It supports the new Elasticsearch 7.x as well as older versions. +7.x has [removed mapping types on indexes](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html), +which resulted in incompatible changes to a number of APIs. ## Documentation @@ -34,7 +43,7 @@ Then run `mix deps.get` to fetch the new dependency. ## Examples -### Creating an Elasticsearch index +### Creating an index ```elixir Elastix.Index.create("http://localhost:9200", "twitter", %{}) @@ -46,9 +55,9 @@ Elastix.Index.create("http://localhost:9200", "twitter", %{}) elastic_url = "http://localhost:9200" data = %{ - user: "kimchy", - post_date: "2009-11-15T14:12:12", - message: "trying out Elastix" + user: "kimchy", + post_date: "2009-11-15T14:12:12", + message: "trying out Elastix" } mapping = %{ @@ -67,7 +76,9 @@ Elastix.Document.delete(elastic_url, "twitter", "tweet", "42") ### Bulk requests -Bulk requests take as parameter a list of the lines you want to send to the [`_bulk`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) endpoint. +Bulk requests take as parameter a list of the lines you want to send to the +[`_bulk`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) +endpoint. You can also specify the following options: @@ -109,7 +120,9 @@ config :elastix, httpoison_options: [hackney: [pool: :elastix_pool]] ``` -Note that you can configure Elastix to use any JSON library, see the ["Custom JSON codec" page](https://hexdocs.pm/elastix/custom-json-codec.html) for more info. +Note that you can configure Elastix to use any JSON library, see the +["Custom JSON codec" page](https://hexdocs.pm/elastix/custom-json-codec.html) for more +info. ### Custom headers @@ -118,7 +131,10 @@ config :elastix, custom_headers: {MyModule, :add_aws_signature, ["us-east"]} ``` -`custom_headers` must be a tuple of the type `{Module, :function, [args]}`, where `:function` is a function that should accept the request (a map of this type: `%{method: String.t, headers: [], url: String.t, body: String.t}`) as its first parameter and return a list of the headers you want to send: +`custom_headers` must be a tuple of the type `{Module, :function, [args]}`, +where `:function` is a function that should accept the request (a map of this +type: `%{method: String.t, headers: [], url: String.t, body: String.t}`) as its +first parameter and return a list of the headers you want to send: ```elixir defmodule MyModule do @@ -134,7 +150,8 @@ end ## Running tests -You need Elasticsearch running locally on port 9200. A quick way of doing so is via Docker: +You need Elasticsearch running locally on port 9200. +A quick way of doing so is via Docker: ``` $ docker run -p 9200:9200 -it --rm elasticsearch:5.1.2 @@ -151,6 +168,8 @@ $ mix test ## Copyright and License -Copyright © 2017 El Werbitzky werbitzky@gmail.com +Copyright © 2017-2019 El Werbitzky werbitzky@gmail.com -This work is free. You can redistribute it and/or modify it under the terms of the Do What The Fuck You Want To Public License, Version 2, as published by Sam Hocevar. See http://www.wtfpl.net/ for more details. +This work is free. You can redistribute it and/or modify it under the terms of +the Do What The Fuck You Want To Public License, Version 2, as published by Sam +Hocevar. See http://www.wtfpl.net/ for more details. diff --git a/lib/elastix.ex b/lib/elastix.ex index 8c5b340..fa67354 100644 --- a/lib/elastix.ex +++ b/lib/elastix.ex @@ -19,4 +19,16 @@ defmodule Elastix do @doc false def config(key, default \\ nil), do: Application.get_env(:elastix, key, default) + + + @doc "Convert Elasticsearch version string into tuple." + def version_to_tuple(version) when is_binary(version) do + version + |> String.split([".", "-"]) + |> Enum.map(&String.to_integer/1) + |> version_to_tuple() + end + def version_to_tuple([first, second]), do: {first, second, 0} + def version_to_tuple([first, second, third | _rest]), do: {first, second, third} + end diff --git a/lib/elastix/alias.ex b/lib/elastix/alias.ex index b8212ce..35fe62a 100644 --- a/lib/elastix/alias.ex +++ b/lib/elastix/alias.ex @@ -1,24 +1,28 @@ defmodule Elastix.Alias do @moduledoc """ - The alias API makes it possible to perform alias operations on indexes. + The alias API adds or removes index aliases, a secondary name used to refer + to one or more existing indices. - [Aliases documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html) + [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Excepts a list of actions for the `actions` parameter. + Perform actions on aliases. + + Specify the list of actions in the `actions` parameter. ## Examples - iex> actions = [%{ add: %{ index: "test1", alias: "alias1" }}] + + iex> actions = [%{add: %{index: "test1", alias: "alias1"}}] iex> Elastix.Alias.post("http://localhost:9200", actions) {:ok, %HTTPoison.Response{...}} + """ - @spec post(elastic_url :: String.t(), actions :: list) :: HTTP.resp() + @spec post(binary, [map]) :: HTTP.resp def post(elastic_url, actions) do - prepare_url(elastic_url, ["_aliases"]) - |> HTTP.post(JSON.encode!(%{actions: actions})) + url = HTTP.make_url(elastic_url, "_aliases") + HTTP.post(url, JSON.encode!(%{actions: actions})) end @doc """ diff --git a/lib/elastix/bulk.ex b/lib/elastix/bulk.ex index 7292d05..3f1b0a8 100644 --- a/lib/elastix/bulk.ex +++ b/lib/elastix/bulk.ex @@ -1,94 +1,69 @@ defmodule Elastix.Bulk do @moduledoc """ - The bulk API makes it possible to perform many index/delete operations in a single API call. + The bulk API performs multiple indexing or delete operations in a single API + call. This reduces overhead and can greatly increase indexing speed. [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} + require Logger + @doc """ - Excepts a list of actions and sources for the `lines` parameter. + Send a batch of actions and sources, encoding them as JSON. + + Data should be encoded as a list of maps. + See the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). ## Examples - iex> Elastix.Bulk.post("http://localhost:9200", [%{index: %{_id: "1"}}, %{user: "kimchy"}], index: "twitter", type: "tweet") + iex> operations = [%{index: %{_id: "1"}}, %{user: "kimchy"}] + iex> Elastix.Bulk.post("http://localhost:9200", operations, index: "twitter", type: "tweet") {:ok, %HTTPoison.Response{...}} - """ - @spec post( - elastic_url :: String.t(), - lines :: list, - opts :: Keyword.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() - def post(elastic_url, lines, options \\ [], query_params \\ []) do - data = - Enum.reduce(lines, [], fn l, acc -> ["\n", JSON.encode!(l) | acc] end) - |> Enum.reverse() - |> IO.iodata_to_binary() - - path = - Keyword.get(options, :index) - |> make_path(Keyword.get(options, :type), query_params) - - httpoison_options = Keyword.get(options, :httpoison_options, []) - elastic_url - |> prepare_url(path) - |> HTTP.put(data, [], httpoison_options) + """ + @spec post(binary, list, Keyword.t, Keyword.t) :: HTTP.resp + def post(elastic_url, operations, options \\ [], query_params \\ []) do + data = for op <- operations, do: [JSON.encode!(op), "\n"] + post_raw(elastic_url, data, options, query_params) end @doc """ - Deprecated: use `post/4` instead. - """ - @spec post_to_iolist( - elastic_url :: String.t(), - lines :: list, - opts :: Keyword.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() - def post_to_iolist(elastic_url, lines, options \\ [], query_params \\ []) do - IO.warn( - "This function is deprecated and will be removed in future releases; use Elastix.Bulk.post/4 instead." - ) + Send a list of actions and sources, with data already encoded as JSON. - httpoison_options = Keyword.get(options, :httpoison_options, []) + Data should be encoded as described in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). - (elastic_url <> - make_path(Keyword.get(options, :index), Keyword.get(options, :type), query_params)) - |> HTTP.put(Enum.map(lines, fn line -> JSON.encode!(line) <> "\n" end), [], httpoison_options) - end + Options: + + * index: Default index name if actions don't specify one + * type: Default type if actions don't specify one. + Type is obsolete in newer versions of Elasticsearch. + See [removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html) + * httpoison_options: options for HTTP call, e.g. setting timeout - @doc """ - Same as `post/4` but instead of sending a list of maps you must send raw binary data in - the format described in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). """ - @spec post_raw( - elastic_url :: String.t(), - raw_data :: String.t(), - opts :: Keyword.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() + @spec post_raw(binary, iodata, Keyword.t, Keyword.t) :: HTTP.resp def post_raw(elastic_url, raw_data, options \\ [], query_params \\ []) do - - httpoison_options = Keyword.get(options, :httpoison_options, []) - - (elastic_url <> - make_path(Keyword.get(options, :index), Keyword.get(options, :type), query_params)) - |> HTTP.put(raw_data, [], httpoison_options) + url = HTTP.make_url(elastic_url, make_path(options[:index], options[:type]), query_params) + headers = [{"Content-Type", "application/x-ndjson"}] + httpoison_options = options[:httpoison_options] || [] + HTTP.post(url, raw_data, headers, httpoison_options) end - @doc false - def make_path(index_name, type_name, query_params) do - path = make_base_path(index_name, type_name) - - case query_params do - [] -> path - _ -> HTTP.append_query_string(path, query_params) - end + @doc deprecated: "Use post/4 instead" + @spec post_to_iolist(binary, list, Keyword.t, Keyword.t) :: HTTP.resp + def post_to_iolist(elastic_url, lines, options \\ [], query_params \\ []) do + Logger.warn("This function is deprecated and will be removed in future releases; use Elastix.Bulk.post/4 instead.") + httpoison_options = options[:httpoison_options] || [] + url = HTTP.make_url(elastic_url, make_path(options[:index], options[:type]), query_params) + HTTP.post(url, Enum.map(lines, fn line -> JSON.encode!(line) <> "\n" end), [], httpoison_options) end - defp make_base_path(nil, nil), do: "/_bulk" - defp make_base_path(index_name, nil), do: "/#{index_name}/_bulk" - defp make_base_path(index_name, type_name), do: "/#{index_name}/#{type_name}/_bulk" + @doc false + # Make path based on index and type options + @spec make_path(binary | nil, binary | nil) :: binary + def make_path(index, type) + def make_path(nil, nil), do: "/_bulk" + def make_path(index, nil), do: "/#{index}/_bulk" + def make_path(index, type), do: "/#{index}/#{type}/_bulk" end diff --git a/lib/elastix/document.ex b/lib/elastix/document.ex index b0a2565..b25fac3 100644 --- a/lib/elastix/document.ex +++ b/lib/elastix/document.ex @@ -4,151 +4,188 @@ defmodule Elastix.Document do [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - (Re)Indexes a document with the given `id`. + Index document. + + * `elastic_url`: base url for Elasticsearch server + * `index`: index name + * `data`: data to index, either a map or binary + * `metadata`: + - id`: document identifier. If not specified, Elasticsearch will generate one. + - `type`: document type + Type is obsolete in newer versions of Elasticsearch. + See [removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html) ## Examples - iex> Elastix.Document.index("http://localhost:9200", "twitter", "tweet", "42", %{user: "kimchy", post_date: "2009-11-15T14:12:12", message: "trying out Elastix"}) + New API: + + iex> data = %{user: "kimchy", post_date: "2009-11-15T14:12:12", message: "trying out Elastix"} + iex> index = "twitter" + iex> Elastix.Document.index("http://localhost:9200", index, data, %{id: "42", type: "tweet"}) {:ok, %HTTPoison.Response{...}} - """ - @spec index( - elastic_url :: String.t(), - index :: String.t(), - type :: String.t(), - id :: String.t(), - data :: map, - query_params :: Keyword.t() - ) :: HTTP.resp() - def index(elastic_url, index_name, type_name, id, data, query_params \\ []) do - prepare_url(elastic_url, make_path(index_name, type_name, query_params, id)) - |> HTTP.put(JSON.encode!(data)) - end - @doc """ - Indexes a new document. + Old API: - ## Examples - - iex> Elastix.Document.index_new("http://localhost:9200", "twitter", "tweet", %{user: "kimchy", post_date: "2009-11-15T14:12:12", message: "trying out Elastix"}) + iex> data = %{user: "kimchy", post_date: "2009-11-15T14:12:12", message: "trying out Elastix"} + iex> index = "twitter" + iex> type = "tweet" + iex> id = "42" + iex> Elastix.Document.index("http://localhost:9200", index, type, id, data) {:ok, %HTTPoison.Response{...}} + """ - @spec index_new( - elastic_url :: String.t(), - index :: String.t(), - type :: String.t(), - data :: map, - query_params :: Keyword.t() - ) :: HTTP.resp() - def index_new(elastic_url, index_name, type_name, data, query_params \\ []) do - prepare_url(elastic_url, make_path(index_name, type_name, query_params)) - |> HTTP.post(JSON.encode!(data)) + # New: @spec index(binary, binary, map | binary, map | binary, Keyword.t) :: HTTP.resp + def index(elastic_url, index, data_or_type, metadata_or_id \\ %{}, data_or_query_params \\ [], query_params \\ []) + + # New: @spec index(binary, binary, binary | map, map, Keyword.t) :: HTTP.resp + def index(elastic_url, index, data, %{id: _} = metadata, query_params, _unused) when is_map(metadata) do + url = HTTP.make_url(elastic_url, make_path(index, metadata), query_params) + HTTP.put(url, JSON.encode!(data)) + end + def index(elastic_url, index, data, metadata, query_params, _unused) when is_map(metadata) do + url = HTTP.make_url(elastic_url, make_path(index, metadata), query_params) + HTTP.post(url, JSON.encode!(data)) # Use post to automatically assign id when not specified + end + + # Old: @spec index(binary, binary, binary, binary, map, Keyword.t) :: HTTP.resp + def index(elastic_url, index, type, id, data, query_params) do + # TODO: Deprecation warning + url = HTTP.make_url(elastic_url, make_path_old(index, type, query_params, id)) + HTTP.put(url, JSON.encode!(data)) + end + + # @doc deprecated: """ + # Index a new document. + # + # ## Examples + # + # iex> data = %{user: "kimchy", post_date: "2009-11-15T14:12:12", message: "trying out Elastix"} + # iex> Elastix.Document.index_new("http://localhost:9200", "twitter", "tweet", data) + # {:ok, %HTTPoison.Response{...}} + # """ + @doc deprecated: "Use index/6 instead" + @spec index_new(binary, binary, binary, map, Keyword.t) :: HTTP.resp + def index_new(elastic_url, index, type, data, query_params \\ []) do + # TODO: deprecation warning + url = HTTP.make_url(elastic_url, make_path_old(index, type, query_params)) + HTTP.post(url, JSON.encode!(data)) end @doc """ - Fetches a document matching the given `id`. + Get document by id. ## Examples + New API without types: + + iex> Elastix.Document.get("http://localhost:9200", "twitter", "42") + {:ok, %HTTPoison.Response{...}} + + Old API with types: + iex> Elastix.Document.get("http://localhost:9200", "twitter", "tweet", "42") {:ok, %HTTPoison.Response{...}} + """ - @spec get( - elastic_url :: String.t(), - index :: String.t(), - type :: String.t(), - id :: String.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() - def get(elastic_url, index_name, type_name, id, query_params \\ []) do - prepare_url(elastic_url, make_path(index_name, type_name, query_params, id)) - |> HTTP.get() + @spec get(binary, binary, binary, Keyword.t | binary, Keyword.t) :: HTTP.resp + + def get(elastic_url, index, id_or_type, query_params_or_id \\ [], query_params \\ []) + + # New API + def get(elastic_url, index, id, query_params, _unused) when is_list(query_params) do + url = HTTP.make_url(elastic_url, make_path(index, %{id: id}), query_params) + HTTP.get(url) end + # Old: @spec get(binary, binary, binary, binary, Keyword.t) :: HTTP.resp + def get(elastic_url, index, type, id, query_params) do + # TODO: deprecation warning + url = HTTP.make_url(elastic_url, make_path_old(index, type, query_params, id)) + HTTP.get(url) + end + + @doc """ - Fetches multiple documents matching the given `query` using the + Get multiple documents with the mget API. + [Multi Get API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html). """ - @spec mget( - elastic_url :: String.t(), - query :: map, - index :: String.t(), - type :: String.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() - def mget(elastic_url, query, index_name \\ nil, type_name \\ nil, query_params \\ []) do - path = - [index_name, type_name] - |> Enum.reject(&is_nil/1) - |> Enum.join("/") - url = - prepare_url(elastic_url, [path, "_mget"]) - |> HTTP.append_query_string(query_params) + @spec mget(binary, map, binary | nil, binary | Keyword.t, Keyword.t) :: HTTP.resp + # Old: @spec mget(binary, map, binary | nil, binary | nil, Keyword.t) :: HTTP.resp + + def mget(elastic_url, query, index \\ nil, query_params_or_type \\ [], query_params \\ []) + + # New API + def mget(elastic_url, query, nil, query_params, _unused) when is_list(query_params) do + do_mget(elastic_url, query, ["_mget"], query_params) + end + def mget(elastic_url, query, index, query_params, _unused) when is_list(query_params) do + do_mget(elastic_url, query, [index, "_mget"], query_params) + end + + # Old API + def mget(elastic_url, query, index, type, query_params) when is_binary(type) do + do_mget(elastic_url, query, [index, type, "_mget"], query_params) + end + defp do_mget(elastic_url, query, path_comps, query_params) do + url = HTTP.make_url(elastic_url, path_comps, query_params) # HTTPoison does not provide an API for a GET request with a body. HTTP.request(:get, url, JSON.encode!(query)) end @doc """ - Deletes the documents matching the given `id`. + Delete the documents matching the given `id`. ## Examples iex> Elastix.Document.delete("http://localhost:9200", "twitter", "tweet", "42") {:ok, %HTTPoison.Response{...}} + """ - @spec delete( - elastic_url :: String.t(), - index :: String.t(), - type :: String.t(), - id :: String.t(), - query_params :: Keyword.t() - ) :: HTTP.resp() - def delete(elastic_url, index_name, type_name, id, query_params \\ []) do - prepare_url(elastic_url, make_path(index_name, type_name, query_params, id)) - |> HTTP.delete() + @spec delete(binary, binary, binary, Keyword.t | binary, Keyword.t) :: HTTP.resp + + def delete(elastic_url, index, type_or_id, id_or_query_params \\ [], query_params \\ []) + + def delete(elastic_url, index, id, query_params, _unused) when is_list(query_params) do + url = HTTP.make_url(elastic_url, make_path(index, %{id: id}), query_params) + HTTP.delete(url) + end + + # Old: @spec delete(binary, binary, binary, binary, Keyword.t) :: HTTP.resp + def delete(elastic_url, index, type, id, query_params) when is_binary(id) or is_integer(id) do + url = HTTP.make_url(elastic_url, make_path_old(index, type, query_params, id)) + HTTP.delete(url) end @doc """ - Deletes the documents matching the given `query` using the + Delete the documents matching the given `query` using the [Delete By Query API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html). """ - @spec delete_matching( - elastic_url :: String.t(), - index :: String.t(), - query :: map, - query_params :: Keyword.t() - ) :: HTTP.resp() - def delete_matching(elastic_url, index_name, %{} = query, query_params \\ []) do - prepare_url(elastic_url, [index_name, "_delete_by_query"]) - |> HTTP.append_query_string(query_params) - |> HTTP.post(JSON.encode!(query)) + @spec delete_matching(binary, binary, map, Keyword.t) :: HTTP.resp + def delete_matching(elastic_url, index, %{} = query, query_params \\ []) do + url = HTTP.make_url(elastic_url, [index, "_delete_by_query"], query_params) + HTTP.post(url, JSON.encode!(query)) end @doc """ - Updates the document with the given `id`. + Update the document with the given `id`. ## Examples - iex> Elastix.Document.update("http://localhost:9200", "twitter", "tweet", "42", %{user: "kimchy", message: "trying out Elastix.Document.update/5"}) + iex> data = %{user: "kimchy", message: "trying out Elastix.Document.update/5"} + iex> Elastix.Document.update("http://localhost:9200", "twitter", "tweet", "42", data) {:ok, %HTTPoison.Response{...}} + """ - @spec update( - elastic_url :: String.t(), - index :: String.t(), - type :: String.t(), - id :: String.t(), - data :: map, - query_params :: Keyword.t() - ) :: HTTP.resp() - def update(elastic_url, index_name, type_name, id, data, query_params \\ []) do - elastic_url - |> prepare_url(make_path(index_name, type_name, query_params, id, "_update")) - |> HTTP.post(JSON.encode!(data)) + @spec update(binary, binary, binary, binary, map, Keyword.t) :: HTTP.resp + def update(elastic_url, index, type, id, data, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_path_old(index, type, query_params, id, "_update")) + HTTP.post(url, JSON.encode!(data)) end @doc """ @@ -180,14 +217,24 @@ defmodule Elastix.Document do end @doc false - def make_path(index_name, type_name, query_params) do - "/#{index_name}/#{type_name}" - |> HTTP.append_query_string(query_params) + @spec make_path(binary, map) :: binary + def make_path(index, metadata \\ %{}) + def make_path(index, %{id: id}), do: "/#{index}/_doc/#{id}" + def make_path(index, %{_id: id}), do: "/#{index}/_doc/#{id}" + def make_path(index, _), do: "/#{index}/_doc/" + + @doc false + def make_path_old(index, type, query_params) do + HTTP.add_query_params("/#{index}/#{type}", query_params) end @doc false - def make_path(index_name, type_name, query_params, id, suffix \\ nil) do - "/#{index_name}/#{type_name}/#{id}/#{suffix}" - |> HTTP.append_query_string(query_params) + def make_path_old(index, type, query_params, id, suffix \\ nil) + def make_path_old(index, type, query_params, id, nil) do + HTTP.add_query_params("/#{index}/#{type}/#{id}", query_params) end + def make_path_old(index, type, query_params, id, suffix) do + HTTP.add_query_params("/#{index}/#{type}/#{id}/#{suffix}", query_params) + end + end diff --git a/lib/elastix/http.ex b/lib/elastix/http.ex index 8f1c00b..0bda3c7 100644 --- a/lib/elastix/http.ex +++ b/lib/elastix/http.ex @@ -1,25 +1,25 @@ defmodule Elastix.HTTP do @moduledoc """ - A thin [HTTPoison](https://github.com/edgurgel/httpoison) wrapper. + A thin wrapper on [HTTPoison](https://github.com/edgurgel/httpoison). """ use HTTPoison.Base alias Elastix.JSON @type resp :: {:ok, HTTPoison.Response.t()} | {:error, HTTPoison.Error.t()} - @doc false - def prepare_url(url, path) when is_binary(path), do: URI.merge(url, path) |> to_string - def prepare_url(url, parts) when is_list(parts), do: prepare_url(url, Path.join(parts)) + @doc "Create URL from base URL, path components and query params." + @spec make_url(binary, binary | [binary], Enum.t) :: binary + def make_url(url, path, query_params \\ []) + def make_url(url, path, query_params) when is_list(path) do + make_url(url, Path.join(path), query_params) + end + def make_url(url, path, query_params) do + URI.merge(url, add_query_params(path, query_params)) |> to_string + end @doc false def request(method, url, body \\ "", headers \\ [], options \\ []) do - query_url = - if Keyword.has_key?(options, :params) do - url <> "?" <> URI.encode_query(options[:params]) - else - url - end - + query_url = add_query_params(url, options[:params]) full_url = to_string(query_url) body = process_request_body(body) @@ -51,32 +51,51 @@ defmodule Elastix.HTTP do end @doc false + @spec process_response_body(binary) :: term def process_response_body(""), do: "" - - def process_response_body(body) do - case body |> to_string |> JSON.decode() do - {:error, _} -> body - {:ok, decoded} -> decoded + def process_response_body(body) when is_binary(body) do + case JSON.decode(body) do + {:error, _} -> + body + {:ok, decoded} -> + decoded end end + def process_response_body(body) do + process_response_body(to_string(body)) + end @doc """ - Encodes an enumerable (`params`) into a query string and appends it to `root`. + Add query parameters to end of path. ## Examples - iex> Elastix.HTTP.append_query_string("/path", %{a: 1, b: 2}) + iex> Elastix.HTTP.add_query_params("/path", a: 1, b: 2) "/path?a=1&b=2" + + iex> Elastix.HTTP.add_query_params("/path", %{a: 1, b: 2}) + "/path?a=1&b=2" + """ - @spec append_query_string(String.t(), term()) :: String.t() - def append_query_string(root, params), do: "#{root}?#{URI.encode_query(params)}" + @spec add_query_params(binary, Enum.t | nil) :: binary + def add_query_params(url, params) + def add_query_params(url, nil), do: url + def add_query_params(url, []), do: url + def add_query_params(url, map) when is_map(map) and map_size(map) == 0, do: url + def add_query_params(url, params), do: url <> "?" <> URI.encode_query(params) defp default_httpoison_options do Elastix.config(:httpoison_options, []) end - defp add_content_type_header(headers) do - [{"Content-Type", "application/json; charset=UTF-8"} | headers] + @doc false + @spec add_content_type_header([{binary, binary}]) :: [{binary, binary}] + def add_content_type_header(headers) do + if :proplists.is_defined("Content-Type", headers) do + headers + else + [{"Content-Type", "application/json; charset=UTF-8"} | headers] + end end defp add_shield_header(headers) do diff --git a/lib/elastix/index.ex b/lib/elastix/index.ex index 0bf208f..a8c7acd 100644 --- a/lib/elastix/index.ex +++ b/lib/elastix/index.ex @@ -1,38 +1,40 @@ defmodule Elastix.Index do @moduledoc """ - The indices APIs are used to manage individual indices, index settings, aliases, mappings, and index templates. + The indices APIs are used to manage individual indices, index settings, + aliases, mappings, and index templates. [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Creates a new index. + Create a new index. ## Examples iex> Elastix.Index.create("http://localhost:9200", "twitter", %{}) {:ok, %HTTPoison.Response{...}} + """ - @spec create(elastic_url :: String.t(), name :: String.t(), data :: map) :: HTTP.resp() - def create(elastic_url, name, data) do - prepare_url(elastic_url, name) - |> HTTP.put(JSON.encode!(data)) + @spec create(binary, binary, map) :: HTTP.resp + def create(elastic_url, index, data) do + url = HTTP.make_url(elastic_url, index) + HTTP.put(url, JSON.encode!(data)) end @doc """ - Deletes an existing index. + Delete an existing index. ## Examples iex> Elastix.Index.delete("http://localhost:9200", "twitter") {:ok, %HTTPoison.Response{...}} + """ - @spec delete(elastic_url :: String.t(), name :: String.t()) :: HTTP.resp() - def delete(elastic_url, name) do - prepare_url(elastic_url, name) - |> HTTP.delete() + @spec delete(binary, binary) :: HTTP.resp + def delete(elastic_url, index) do + url = HTTP.make_url(elastic_url, index) + HTTP.delete(url) end @doc """ @@ -42,82 +44,89 @@ defmodule Elastix.Index do iex> Elastix.Index.get("http://localhost:9200", "twitter") {:ok, %HTTPoison.Response{...}} + """ - @spec get(elastic_url :: String.t(), name :: String.t()) :: HTTP.resp() - def get(elastic_url, name) do - prepare_url(elastic_url, name) - |> HTTP.get() + @spec get(binary, binary) :: HTTP.resp + def get(elastic_url, index) do + url = HTTP.make_url(elastic_url, index) + HTTP.get(url) end @doc """ - Returns `true` if the specified index exists, `false` otherwise. + Check if index exists. + + Returns `{:ok, true}` if the index exists. ## Examples iex> Elastix.Index.exists?("http://localhost:9200", "twitter") {:ok, false} + iex> Elastix.Index.create("http://localhost:9200", "twitter", %{}) {:ok, %HTTPoison.Response{...}} + iex> Elastix.Index.exists?("http://localhost:9200", "twitter") {:ok, true} + """ - @spec exists?(elastic_url :: String.t(), name :: String.t()) :: - {:ok, boolean()} | {:error, HTTPoison.Error.t()} - def exists?(elastic_url, name) do - case prepare_url(elastic_url, name) |> HTTP.head() do - {:ok, response} -> - case response.status_code do - 200 -> {:ok, true} - 404 -> {:ok, false} - end - - err -> - err + @spec exists?(binary, binary) :: {:ok, boolean} | {:error, HTTPoison.Error.t} + def exists?(elastic_url, index) do + url = HTTP.make_url(elastic_url, index) + case HTTP.head(url) do + {:ok, %{status_code: 200}} -> {:ok, true} + {:ok, %{status_code: 404}} -> {:ok, false} + err -> err end end @doc """ - Forces the [refresh](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html) - of the specified index. + Force refresh of index. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html) ## Examples iex> Elastix.Index.refresh("http://localhost:9200", "twitter") {:ok, %HTTPoison.Response{...}} + """ - @spec refresh(elastic_url :: String.t(), name :: String.t()) :: HTTP.resp() - def refresh(elastic_url, name) do - prepare_url(elastic_url, [name, "_refresh"]) - |> HTTP.post("") + @spec refresh(binary, binary) :: HTTP.resp + def refresh(elastic_url, index) do + url = HTTP.make_url(elastic_url, [index, "_refresh"]) + HTTP.post(url, "") end @doc """ - [Opens](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html) - the specified index. + Open index. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html) ## Examples iex> Elastix.Index.open("http://localhost:9200", "twitter") {:ok, %HTTPoison.Response{...}} + """ - @spec open(elastic_url :: String.t(), name :: String.t()) :: HTTP.resp() - def open(elastic_url, name) do - prepare_url(elastic_url, [name, "_open"]) - |> HTTP.post("") + @spec open(binary, binary) :: HTTP.resp + def open(elastic_url, index) do + url = HTTP.make_url(elastic_url, [index, "_open"]) + HTTP.post(url, "") end @doc """ - [Closes](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html) - the specified index. + Close index. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html) ## Examples iex> Elastix.Index.close("http://localhost:9200", "twitter") {:ok, %HTTPoison.Response{...}} + """ - @spec close(elastic_url :: String.t(), name :: String.t()) :: HTTP.resp() - def close(elastic_url, name) do - prepare_url(elastic_url, [name, "_close"]) - |> HTTP.post("") + @spec close(binary, binary) :: HTTP.resp + def close(elastic_url, index) do + url = HTTP.make_url(elastic_url, [index, "_close"]) + HTTP.post(url, "") end end diff --git a/lib/elastix/json.ex b/lib/elastix/json.ex index 26c6e23..02af584 100644 --- a/lib/elastix/json.ex +++ b/lib/elastix/json.ex @@ -1,4 +1,7 @@ defmodule Elastix.JSON do + + require Logger + defmodule Codec do @moduledoc """ A behaviour for JSON serialization. @@ -53,7 +56,7 @@ defmodule Elastix.JSON do Elastix.config(:json_options, []) opts -> - IO.warn( + Logger.warn( "Using :poison_options is deprecated and might not work in future releases; use :json_options instead." ) diff --git a/lib/elastix/mapping.ex b/lib/elastix/mapping.ex index aa24ad6..03d9dde 100644 --- a/lib/elastix/mapping.ex +++ b/lib/elastix/mapping.ex @@ -2,143 +2,161 @@ defmodule Elastix.Mapping do @moduledoc """ The mapping API is used to define how documents are stored and indexed. - [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html) + [Elastic docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html) + + [Removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html) + """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Creates a new mapping. + Add field to an existing mapping. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html#add-field-mapping) ## Examples - iex> mapping = %{properties: %{user: %{type: "text"}, post_date: %{type: "date"}, message: %{type: "text"}}} - iex> Elastix.Mapping.put("http://localhost:9200", "twitter", "tweet", mapping) + New API + + iex> mappings = %{properties: %{user: %{type: "text"}, post_date: %{type: "date"}, message: %{type: "text"}}} + iex> Elastix.Mapping.put("http://localhost:9200", "twitter", mappings) {:ok, %HTTPoison.Response{...}} + + Old API with mapping types + + iex> mappings = %{properties: %{user: %{type: "text"}, post_date: %{type: "date"}, message: %{type: "text"}}} + iex> Elastix.Mapping.put("http://localhost:9200", "twitter", "tweet", mappings) + {:ok, %HTTPoison.Response{...}} + """ - @spec put( - elastic_url :: String.t(), - index_names :: String.t() | list, - type_name :: String.t(), - data :: map, - query_params :: Keyword.t() - ) :: HTTP.resp() - def put(elastic_url, index_names, type_name, data, query_params \\ []) - - def put(elastic_url, index_names, type_name, data, query_params) - when is_list(index_names) do - prepare_url(elastic_url, make_path(index_names, [type_name], query_params)) - |> HTTP.put(JSON.encode!(data)) + # Support the old Elasticsearch API with mapping types and the new API + # without mapping types using the same natural function name. + @spec put(binary, binary | [binary], binary | map, map | Keyword.t, Keyword.t) :: HTTP.resp + + def put(elastic_url, index, type_or_mappings, mappings_or_query_params \\ [], query_params \\ []) + + # Old: @spec put(binary, binary | [binary], binary, map, Keyword.t) :: HTTP.resp + def put(elastic_url, indexes, type, mappings, query_params) when is_binary(type) do + url = HTTP.make_url(elastic_url, put_path(indexes, type, mappings, query_params)) + HTTP.put(url, JSON.encode!(mappings)) end - def put(elastic_url, index_name, type_name, data, query_params), - do: put(elastic_url, [index_name], type_name, data, query_params) + # New: @spec put(binary, binary, map, Keyword.t) :: HTTP.resp + def put(elastic_url, index, mappings, query_params, unused) when is_map(mappings) do + # New API + url = HTTP.make_url(elastic_url, put_path(index, mappings, query_params, unused)) + HTTP.put(url, JSON.encode!(mappings)) + end + + @doc false + @spec put_path(binary | [binary], binary | map, map | Keyword.t, Keyword.t) :: binary + def put_path(indexes, type_or_mappings, mappings_or_query_params \\ [], query_params \\ []) + # Old + def put_path(indexes, type, mappings, query_params) when is_list(indexes) and is_binary(type) do + indexes = Enum.join(indexes, ",") + put_path(indexes, type, mappings, query_params) + end + def put_path(indexes, type, _mappings, query_params) when is_binary(type) do + path = "/#{indexes}/_mapping/#{type}" + HTTP.add_query_params(path, query_params) + end + # New + def put_path(index, mappings, query_params, _unused) when is_map(mappings) do + path = "/#{index}/_mapping" + HTTP.add_query_params(path, query_params) + end @doc """ - Gets info on one or a list of mappings for one or a list of indices. + Get mappings for index. + + [New Elasticsearch API](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/mapping.html) + + [Old Elasticsearch API](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/mapping.html) + + Accepts a single index or list of indexes. + Accepts a single type or list of types. + + https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html#view-mapping ## Examples + iex> Elastix.Mapping.get("http://localhost:9200", "twitter") + {:ok, %HTTPoison.Response{...}} + iex> Elastix.Mapping.get("http://localhost:9200", "twitter", "tweet") {:ok, %HTTPoison.Response{...}} + """ - @spec get( - elastic_url :: String.t(), - index_names :: String.t() | list, - type_names :: String.t() | list, - query_params :: Keyword.t() - ) :: HTTP.resp() - def get(elastic_url, index_names, type_names, query_params \\ []) - - def get(elastic_url, index_names, type_names, query_params) - when is_list(type_names) and is_list(index_names) do - prepare_url(elastic_url, make_path(index_names, type_names, query_params)) - |> HTTP.get() + @spec get(binary, binary | [binary], binary | [binary] | Keyword.t, Keyword.t) :: HTTP.resp + # New: @spec get(binary, binary, Keyword.t, Keyword.t) :: HTTP.resp + # Old: @spec get(binary, binary | [binary], binary | [binary], Keyword.t) :: HTTP.resp + def get(elastic_url, indexes, types_or_query_params \\ [], query_params \\ []) do + url = HTTP.make_url(elastic_url, get_path(indexes, types_or_query_params, query_params)) + HTTP.get(url) end - def get(elastic_url, index_names, type_name, query_params) - when is_list(index_names) do - get(elastic_url, index_names, [type_name], query_params) + @spec get_path(binary | [binary], binary | [binary] | Keyword.t, Keyword.t) :: binary + # New: @spec get(binary, Keyword.t, Keyword.t) :: binary + def get_path(index, [], []) when is_binary(index) do + "/#{index}/_mapping" end - - def get(elastic_url, index_name, type_names, query_params) - when is_list(type_names) do - get(elastic_url, [index_name], type_names, query_params) + def get_path(index, [value | _rest] = query_params, []) when is_binary(index) and is_tuple(value) do + HTTP.add_query_params("/#{index}/_mapping", query_params) end - def get(elastic_url, index_name, type_name, query_params), - do: get(elastic_url, [index_name], [type_name], query_params) + # Old: @spec get(binary | [binary], binary | [binary], Keyword.t) :: binary + def get_path(indexes, [value | _rest] = types, query_params) when is_list(indexes) and is_binary(value) do + indexes = Enum.join(indexes, ",") + types = Enum.join(types, ",") + HTTP.add_query_params("/#{indexes}/_mapping/#{types}", query_params) + end + def get_path(indexes, type, query_params) when is_list(indexes) and is_binary(type) do + indexes = Enum.join(indexes, ",") + HTTP.add_query_params("/#{indexes}/_mapping/#{type}", query_params) + end + def get_path(index, [value | _rest] = types, query_params) when is_binary(index) and is_binary(value) do + types = Enum.join(types, ",") + HTTP.add_query_params("/#{index}/_mapping/#{types}", query_params) + end + def get_path(index, type, query_params) when is_binary(index) and is_binary(type) do + HTTP.add_query_params("/#{index}/_mapping/#{type}", query_params) + end @doc """ - Gets info on every mapping. + Get info on all mappings. ## Examples iex> Elastix.Mapping.get_all("http://localhost:9200") {:ok, %HTTPoison.Response{...}} + """ - @spec get_all(elastic_url :: String.t(), query_params :: Keyword.t()) :: HTTP.resp() + @spec get_all(binary, Keyword.t) :: HTTP.resp def get_all(elastic_url, query_params \\ []) do - prepare_url(elastic_url, make_all_path(query_params)) - |> HTTP.get() + url = HTTP.make_url(elastic_url, "_mapping", query_params) + HTTP.get(url) end @doc """ - Gets info on every given mapping. + Get info on all mappings for types. ## Examples - iex> Elastix.Mapping.get_all("http://localhost:9200", ["tweet", "user"]) + iex> Elastix.Mapping.get_all_with_type("http://localhost:9200", ["tweet", "user"]) {:ok, %HTTPoison.Response{...}} - """ - @spec get_all_with_type( - elastic_url :: String.t(), - type_names :: String.t() | list, - query_params :: Keyword.t() - ) :: HTTP.resp() - def get_all_with_type(elastic_url, type_names, query_params \\ []) - - def get_all_with_type(elastic_url, type_names, query_params) - when is_list(type_names) do - prepare_url(elastic_url, make_all_path(type_names, query_params)) - |> HTTP.get() - end - def get_all_with_type(elastic_url, type_name, query_params), - do: get_all_with_type(elastic_url, [type_name], query_params) - - @doc false - def make_path(index_names, type_names, query_params) do - index_names = Enum.join(index_names, ",") - type_names = Enum.join(type_names, ",") - - path = "/#{index_names}/_mapping/#{type_names}" - - case query_params do - [] -> path - _ -> HTTP.append_query_string(path, query_params) - end + """ + @spec get_all_with_type(binary, [binary], Keyword.t) :: HTTP.resp + def get_all_with_type(elastic_url, types, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_all_path(types), query_params) + HTTP.get(url) end @doc false - def make_all_path(query_params) do - path = "/_mapping" - - case query_params do - [] -> path - _ -> HTTP.append_query_string(path, query_params) - end + @spec make_all_path([binary]) :: binary + def make_all_path(types) when is_list(types) do + types = Enum.join(types, ",") + "/_mapping/#{types}" end - @doc false - def make_all_path(type_names, query_params) do - type_names = Enum.join(type_names, ",") - - path = "/_mapping/#{type_names}" - - case query_params do - [] -> path - _ -> HTTP.append_query_string(path, query_params) - end - end end diff --git a/lib/elastix/search.ex b/lib/elastix/search.ex index 7b22260..be6191f 100644 --- a/lib/elastix/search.ex +++ b/lib/elastix/search.ex @@ -4,133 +4,116 @@ defmodule Elastix.Search do [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Makes a request to the `_search` or the `_msearch` endpoint depending on the type of - `data`. + Search index based on query. - When passing a map for data, it'll make a simple search, but you can pass a list of + Query can be specified as a single map or as a list of maps. + + If a list, uses the [multi search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html) + to execute several searches from a single API request. + + When passing a map for data, makes a simple search, but you can pass a list of header and body params to make a [multi search](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html). ## Examples - iex> Elastix.Search.search("http://localhost:9200", "twitter", ["tweet"], %{query: %{term: %{user: "kimchy"}}}) + iex> query = %{query: %{term: %{user: "kimchy"}}} + iex> Elastix.Search.search("http://localhost:9200", "twitter", ["tweet"], query) {:ok, %HTTPoison.Response{...}} - iex> Elastix.Search.search("http://localhost:9200", ["twitter", "other-index"], ["tweet"], %{query: %{term: %{user: "kimchy"}}}) + iex> query = %{query: %{term: %{user: "kimchy"}}} + iex> Elastix.Search.search("http://localhost:9200", "twitter", [], query) {:ok, %HTTPoison.Response{...}} - """ - @spec search( - elastic_url :: String.t(), - index :: String.t() | list, - types :: list, - data :: map | list - ) :: HTTP.resp() - def search(elastic_url, index, types, data) when is_list(data), - do: search(elastic_url, index, types, data, []) - def search(elastic_url, index, types, data), - do: search(elastic_url, index, types, data, []) + """ + @spec search(elastic_url :: String.t(), index :: String.t() | list, types :: list, data :: map | list) :: HTTP.resp() + def search(elastic_url, index, types, query) do + search(elastic_url, index, types, query, []) + end @doc """ - Same as `search/4` but allows to specify query params and options for - [`HTTPoison.request/5`](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + Search index based on query, with query params and HTTP options. + + See [`HTTPoison.request/5`](https://hexdocs.pm/httpoison/HTTPoison.html#request/5) for options. """ @spec search( - elastic_url :: String.t(), - index :: String.t() | list, - types :: list, - data :: map | list, - query_params :: Keyword.t(), + elastic_url :: String.t(), + index :: String.t(), + types :: list, + data :: map | list, + query_params :: Keyword.t(), options :: Keyword.t() ) :: HTTP.resp() - def search(elastic_url, index, types, data, query_params, options \\ []) - - def search(elastic_url, index, types, data, query_params, options) - when is_list(index) do - search(elastic_url, Enum.join(index, ","), types, data, query_params, options) - end - - def search(elastic_url, index, types, data, query_params, options) - when is_list(data) do - data = - Enum.reduce(data, [], fn d, acc -> ["\n", JSON.encode!(d) | acc] end) - |> Enum.reverse() - |> IO.iodata_to_binary() + def search(elastic_url, index, types, query, query_params, httpoison_options \\ []) - prepare_url(elastic_url, make_path(index, types, query_params, "_msearch")) - |> HTTP.post(data, [], options) + def search(elastic_url, index, types, query, query_params, httpoison_options) when is_list(query) do + url = HTTP.make_url(elastic_url, make_path(index, types, "_msearch"), query_params) + data = for q <- query, do: [JSON.encode!(q), "\n"] + headers = [{"Content-Type", "application/x-ndjson"}] + HTTP.post(url, data, headers, httpoison_options) end - def search(elastic_url, index, types, data, query_params, options) do - prepare_url(elastic_url, make_path(index, types, query_params)) - |> HTTP.post(JSON.encode!(data), [], options) + def search(elastic_url, index, types, query, query_params, httpoison_options) when is_map(query) do + url = HTTP.make_url(elastic_url, make_path(index, types), query_params) + HTTP.post(url, JSON.encode!(query), [], httpoison_options) end @doc """ - Uses the [Scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html) - to allow scrolling through a list of results. + Search with scrolling through results. + + See the [Scroll API docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html). ## Examples - iex> Elastix.Search.scroll("http://localhost:9200", %{query: %{term: %{user: "kimchy"}}}) + iex> query = %{query: %{term: %{user: "kimchy"}}}, scroll: "1m") + iex> {:ok, response} = Elastix.Search.search("http://localhost:9200", "twitter", [], query, scroll: "1m") + iex> scroll_id = response.body["_scroll_id"] + iex> params = %{scroll: "1m", scroll_id: scroll_id} + iex> Elastix.Search.scroll("http://localhost:9200", params) {:ok, %HTTPoison.Response{...}} + """ - @spec scroll(elastic_url :: String.t(), data :: map, options :: Keyword.t()) :: - HTTP.resp() - def scroll(elastic_url, data, options \\ []) do - prepare_url(elastic_url, "_search/scroll") - |> HTTP.post(JSON.encode!(data), [], options) + @spec scroll(binary, map, Keyword.t) :: HTTP.resp + def scroll(elastic_url, scroll_params, httpoison_options \\ []) do + url = HTTP.make_url(elastic_url, "_search/scroll") + HTTP.post(url, JSON.encode!(scroll_params), [], httpoison_options) end @doc """ - Returns the number of results for a query using the - [Count API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html). + Returns the number of results for a query using count API. + + See [Count API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html). ## Examples iex> Elastix.Search.count("http://localhost:9200", "twitter", ["tweet"], %{query: %{term: %{user: "kimchy"}}}) {:ok, %HTTPoison.Response{...}} + """ - @spec count(elastic_url :: String.t(), index :: String.t(), types :: list, data :: map) :: - HTTP.resp() - def count(elastic_url, index, types, data), - do: count(elastic_url, index, types, data, []) + @spec count(binary, binary, list, map) :: HTTP.resp + def count(elastic_url, index, types, data) do + count(elastic_url, index, types, data, []) + end @doc """ - Same as `count/4` but allows to specify query params and options for - [`HTTPoison.request/5`](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + Returns the number of results for a query using count API, supporting query params and HTTP options. + + See [`HTTPoison.request/5`](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). """ - @spec count( - elastic_url :: String.t(), - index :: String.t(), - types :: list, - data :: map, - query_params :: Keyword.t(), - options :: Keyword.t() - ) :: HTTP.resp() + @spec count(binary, binary, list, map, Keyword.t, Keyword.t) :: HTTP.resp def count(elastic_url, index, types, data, query_params, options \\ []) do - (elastic_url <> make_path(index, types, query_params, "_count")) - |> HTTP.post(JSON.encode!(data), [], options) + url = HTTP.make_url(elastic_url, make_path(index, types, "_count"), query_params) + HTTP.post(url, JSON.encode!(data), [], options) end @doc false - def make_path(index, types, query_params, api_type \\ "_search") do - path_root = "/#{index}" - - path = - case types do - [] -> path_root - _ -> path_root <> "/" <> Enum.join(types, ",") - end - - full_path = "#{path}/#{api_type}" - - case query_params do - [] -> full_path - _ -> HTTP.append_query_string(full_path, query_params) - end + @spec make_path(binary, [binary], binary) :: binary + def make_path(index, types, api_type \\ "_search") + def make_path(index, [], api_type), do: "/#{index}/#{api_type}" + def make_path(index, types, api_type) do + types = Enum.join(types, ",") + "/#{index}/#{types}/#{api_type}" end end diff --git a/lib/elastix/snapshot/repository.ex b/lib/elastix/snapshot/repository.ex index 2280e28..aec1f4f 100644 --- a/lib/elastix/snapshot/repository.ex +++ b/lib/elastix/snapshot/repository.ex @@ -1,76 +1,257 @@ defmodule Elastix.Snapshot.Repository do @moduledoc """ - Functions for working with repositories. A repository is required for taking and restoring snapshots of indices. + Functions for working with repositories. + + A repository is a location used to store snapshots when + backing up and restoring data in an Elasticsearch cluster. [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Registers a repository. + Register repository. + + It's necessary to register a snapshot repository before performing + snapshot and restore operations. + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> config = %{type: "fs", settings: %{location: "/tmp/elastix/backups"}} + iex> Elastix.Repository.register(elastic_url, repository, config) + {:ok, + %HTTPoison.Response{ + body: %{"acknowledged" => true}, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "21"}], + request: %HTTPoison.Request{ + body: "{\"type\":\"fs\",\"settings\":{\"location\":\"/tmp/elastix/backups\"}}", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :put, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1", + status_code: 200 + } + } + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_2" + iex> config = %{type: "fs", settings: %{location: "/tmp/elastix/backups"}} + iex> Elastix.Repository.register(elastic_url, repository, config, verify: false) + {:ok, + %HTTPoison.Response{ + body: %{"acknowledged" => true}, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "21"}], + request: %HTTPoison.Request{ + body: "{\"type\":\"fs\",\"settings\":{\"location\":\"/tmp/elastix/backups\"}}", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :put, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_2?verify=false" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_2?verify=false", + status_code: 200 + } + } + """ - @spec register(String.t(), String.t(), Map.t(), [tuple()]) :: - {:ok, %HTTPoison.Response{}} - def register(elastic_url, repo_name, data, query_params \\ []) do - elastic_url - |> prepare_url(make_path(repo_name, query_params)) - |> HTTP.put(JSON.encode!(data)) + @spec register(binary, binary, map, Keyword.t) :: HTTP.resp + def register(elastic_url, repo, config, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_path(repo), query_params) + HTTP.put(url, JSON.encode!(config)) end @doc """ - Verifies a registered but unverified repository. + Verify repository. + + Verify a repository which was initially registered without verification, + (`verify: false`). + + Returns list of nodes where repository was successfully verified or an error + message if verification failed. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#_repository_verification) + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository" + iex> Elastix.Repository.verify(elastic_url, repository) + {:ok, + %HTTPoison.Response{ + body: %{"nodes" => %{"O5twn2YcS0GvFPra5lllUQ" => %{"name" => "O5twn2Y"}}}, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "55"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :post, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_2/_verify" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_2/_verify", + status_code: 200 + } + } + """ - @spec verify(String.t(), String.t()) :: {:ok, %HTTPoison.Response{}} - def verify(elastic_url, repo_name) do - elastic_url - |> prepare_url([make_path(repo_name), "_verify"]) - |> HTTP.post("") + @spec verify(binary, binary) :: HTTP.resp + def verify(elastic_url, repo) do + url = HTTP.make_url(elastic_url, [make_path(repo), "_verify"]) + HTTP.post(url, "") end @doc """ - If repo_name specified, will retrieve information about a registered repository. - Otherwise, will retrieve information about all repositories. + Get info about repositories. + + Gets info about all repositories, or a single repo if specified. + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> Elastix.Repository.get(elastic_url) + {:ok, + %HTTPoison.Response{ + body: %{ + "elastix_test_repository_1" => %{"settings" => %{"location" => "/tmp/elastix/backups"}, "type" => "fs"}, + "elastix_test_repository_2" => %{"settings" => %{"location" => "/tmp/elastix/backups"}, "type" => "fs"} + }, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "179"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :get, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/_all" + }, + request_url: "http://127.0.0.1:9200/_snapshot/_all", + status_code: 200 + } + } + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> Elastix.Repository.get(elastic_url, repository) + {:ok, + %HTTPoison.Response{ + body: %{"elastix_test_repository_1" => %{"settings" => %{"location" => "/tmp/elastix/backups"}, "type" => "fs"}}, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "90"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :get, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1", + status_code: 200 + } + } + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix*" + iex> Elastix.Repository.get(elastic_url, repository) + + iex> elastic_url = "http://localhost:9200" + iex> repository = "foo,bar" + iex> Elastix.Repository.get(elastic_url, repository) + + iex> elastic_url = "http://localhost:9200" + iex> repository = "nonexistent" + iex> Elastix.Repository.get(elastic_url, repository) + {:ok, + %HTTPoison.Response{ + body: %{ + "error" => %{ + "reason" => "[nonexistent] missing", + "root_cause" => [%{"reason" => "[nonexistent] missing", "type" => "repository_missing_exception"}], + "type" => "repository_missing_exception" + }, + "status" => 404 + }, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "183"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :get, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/nonexistent" + }, + request_url: "http://127.0.0.1:9200/_snapshot/nonexistent", + status_code: 404 + } + } + + """ - @spec get(String.t(), String.t()) :: {:ok, %HTTPoison.Response{}} - def get(elastic_url, repo_name \\ "_all") do - elastic_url - |> prepare_url(make_path(repo_name)) - |> HTTP.get() + @spec get(binary, binary) :: HTTP.resp + def get(elastic_url, repo \\ "_all") do + url = HTTP.make_url(elastic_url, make_path(repo)) + HTTP.get(url) end + @doc """ - Removes the reference to the location where the snapshots are stored. + Remove reference to location where snapshots are stored. + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> Elastix.Repository.get(elastic_url, repository) + {:ok, + %HTTPoison.Response{ + body: %{"acknowledged" => true}, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "21"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :delete, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository_1", + status_code: 200 + } + } + """ - @spec delete(String.t(), String.t()) :: {:ok, %HTTPoison.Response{}} - def delete(elastic_url, repo_name) do - elastic_url - |> prepare_url(make_path(repo_name)) - |> HTTP.delete() + @spec delete(binary, binary) :: HTTP.resp + def delete(elastic_url, repo) do + url = HTTP.make_url(elastic_url, make_path(repo)) + HTTP.delete(url) end - @doc false - @spec make_path(String.t(), [tuple()]) :: String.t() - def make_path(repo_name, query_params \\ []) do - path = _make_base_path(repo_name) - - case query_params do - [] -> path - _ -> _add_query_params(path, query_params) - end - end + @doc """ + Clean up unreferenced data in a repository. - defp _make_base_path(nil), do: "/_snapshot" - defp _make_base_path(repo_name), do: "/_snapshot/#{repo_name}" + Trigger a complete accounting of the repositories contents and subsequent + deletion of all unreferenced data that was found. - defp _add_query_params(path, query_params) do - query_string = - query_params - |> Enum.map_join("&", fn param -> - "#{elem(param, 0)}=#{elem(param, 1)}" - end) + Deleting a snapshot performs this cleanup. - "#{path}?#{query_string}" + Available in Elasticsearch 7.x and later. + """ + @spec cleanup(binary, binary) :: HTTP.resp + def cleanup(elastic_url, repo) do + url = HTTP.make_url(elastic_url, [make_path(repo), "_cleanup"]) + HTTP.post(url, "") end + + @doc false + # Make path from arguments + @spec make_path(binary | nil) :: binary + def make_path(nil), do: "/_snapshot" + def make_path(repo), do: "/_snapshot/#{repo}" + end diff --git a/lib/elastix/snapshot/snapshot.ex b/lib/elastix/snapshot/snapshot.ex index 1da3ebc..72ad962 100644 --- a/lib/elastix/snapshot/snapshot.ex +++ b/lib/elastix/snapshot/snapshot.ex @@ -5,58 +5,160 @@ defmodule Elastix.Snapshot.Snapshot do [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html) """ - import Elastix.HTTP, only: [prepare_url: 2] alias Elastix.{HTTP, JSON} @doc """ - Creates a snapshot. + Create snapshot. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#snapshots-take-snapshot) + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> snapshot = "elastix_test_snapshot_2" + iex> config = %{indices: "elastix_test_index_1,elastix_test_index_2" + iex> Elastix.Snapshot.create(elastic_url, repository, snapshot, wait_for_completion: true) + {:ok, + %HTTPoison.Response{ + body: %{ + "snapshot" => %{ + "duration_in_millis" => 74, + "end_time" => "2019-11-17T00:29:44.931Z", + "end_time_in_millis" => 1573950584931, + "failures" => [], + "include_global_state" => true, + "indices" => ["elastix_test_index_2", "elastix_test_index_1"], + "shards" => %{"failed" => 0, "successful" => 10, "total" => 10}, + "snapshot" => "elastix_test_snapshot_2", + "start_time" => "2019-11-17T00:29:44.857Z", + "start_time_in_millis" => 1573950584857, + "state" => "SUCCESS", + "uuid" => "kBL1rleOQS-qfXqvXatNng", + "version" => "6.8.4", + "version_id" => 6080499 + } + }, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "463"}], + request: %HTTPoison.Request{ + body: "{\"indices\":\"elastix_test_index_1,elastix_test_index_2\"}", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :put, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository/elastix_test_snapshot_2?wait_for_completion=true" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository/elastix_test_snapshot_2?wait_for_completion=true", + status_code: 200 + } + } + """ - @spec create(String.t(), String.t(), String.t(), Map.t(), [tuple()], Keyword.t()) :: - {:ok, %HTTPoison.Response{}} - def create(elastic_url, repo_name, snapshot_name, data \\ %{}, query_params \\ [], options \\ []) do - elastic_url - |> prepare_url(make_path(repo_name, snapshot_name, query_params)) - |> HTTP.put(JSON.encode!(data), [], _make_httpoison_options(options)) + + @spec create(binary, binary, binary, map, Keyword.t, Keyword.t) :: HTTP.resp + def create(elastic_url, repo, snapshot, config \\ %{}, query_params \\ [], options \\ []) do + url = HTTP.make_url(elastic_url, make_path(repo, snapshot), query_params) + HTTP.put(url, JSON.encode!(config), Keyword.get(options, :httpoison_options, [])) end @doc """ - Restores a previously created snapshot. + Restore previously created snapshot. """ - @spec restore(String.t(), String.t(), String.t(), Map.t(), Keyword.t()) :: - {:ok, %HTTPoison.Response{}} - def restore(elastic_url, repo_name, snapshot_name, data \\ %{}, options \\ []) do - elastic_url - |> prepare_url([make_path(repo_name, snapshot_name), "_restore"]) - |> HTTP.post(JSON.encode!(data), [], _make_httpoison_options(options)) + @spec restore(binary, binary, binary, map, Keyword.t) :: HTTP.resp + def restore(elastic_url, repo, snapshot, data \\ %{}, options \\ []) do + url = HTTP.make_url(elastic_url, [make_path(repo, snapshot), "_restore"]) + HTTP.post(url, JSON.encode!(data), Keyword.get(options, :httpoison_options, [])) end @doc """ - If repo_name and snapshot_name is specified, will retrieve the status of that - snapsot. If repo_name is specified, will retrieve the status of all snapshots + Get status of snapshots. + + If repo and snapshot is specified, will retrieve the status of that + snapshot. If repo is specified, will retrieve the status of all snapshots in that repository. Otherwise, will retrieve the status of all snapshots. + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> snapshot = "elastix_test_snapshot_2" + iex> Elastix.Snapshot.status(elastic_url, repository, snapshot) + """ - @spec status(String.t(), String.t(), String.t(), Keyword.t()) :: {:ok, %HTTPoison.Response{}} - def status(elastic_url, repo_name \\ "", snapshot_name \\ "", options \\ []) do - elastic_url - |> prepare_url([make_path(repo_name, snapshot_name), "_status"]) - |> HTTP.get([], _make_httpoison_options(options)) + @spec status(binary, binary, binary) :: HTTP.resp + def status(elastic_url, repo \\ "", snapshot \\ "") do + url = HTTP.make_url(elastic_url, [make_path(repo, snapshot), "_status"]) + HTTP.get(url, [], Keyword.get(options, :httpoison_options, [])) end @doc """ + Get information about snapshot. + If repo_name and snapshot_name is specified, will retrieve information about that snapshot. If repo_name is specified, will retrieve information about all snapshots in that repository. Otherwise, will retrieve information about all snapshots. + + ## Examples + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> snapshot = "elastix_test_snapshot_2" + iex> Elastix.Snapshot.get(elastic_url, repository, snapshot) + {:ok, + %HTTPoison.Response{ + body: %{ + "snapshots" => [ + %{ + "duration_in_millis" => 45, + "end_time" => "2019-11-17T00:37:03.858Z", + "end_time_in_millis" => 1573951023858, + "failures" => [], + "include_global_state" => true, + "indices" => ["elastix_test_index_2", "elastix_test_index_1"], + "shards" => %{"failed" => 0, "successful" => 10, "total" => 10}, + "snapshot" => "elastix_test_snapshot_2", + "start_time" => "2019-11-17T00:37:03.813Z", + "start_time_in_millis" => 1573951023813, + "state" => "SUCCESS", + "uuid" => "_l_J5caMQkWVx16kLhwoaw", + "version" => "6.8.4", + "version_id" => 6080499 + } + ] + }, + headers: [{"content-type", "application/json; charset=UTF-8"}, {"content-length", "466"}], + request: %HTTPoison.Request{ + body: "", + headers: [{"Content-Type", "application/json; charset=UTF-8"}], + method: :get, + options: [], + params: %{}, + url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository/elastix_test_snapshot_2" + }, + request_url: "http://127.0.0.1:9200/_snapshot/elastix_test_repository/elastix_test_snapshot_2", + status_code: 200 + } + } + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> Elastix.Snapshot.get(elastic_url, repository) + + iex> elastic_url = "http://localhost:9200" + iex> repository = "elastix_test_repository_1" + iex> Elastix.Snapshot.get(elastic_url) + """ - @spec get(String.t(), String.t(), String.t(), Keyword.t()) :: {:ok, %HTTPoison.Response{}} + + @spec get(binary, binary, binary) :: HTTP.resp def get(elastic_url, repo_name \\ "", snapshot_name \\ "_all", options \\ []) do - elastic_url - |> prepare_url(make_path(repo_name, snapshot_name)) - |> HTTP.get([], _make_httpoison_options(options)) + url = HTTP.make_url(elastic_url, make_path(repo_name, snapshot_name)) + HTTP.get(url, [], Keyword.get(options, :httpoison_options, [])) end @doc """ - Deletes a snapshot from a repository. + Delete snapshot from repository. This can also be used to stop currently running snapshot and restore operations. Snapshot deletes can be slow, so you can pass in @@ -76,32 +178,18 @@ defmodule Elastix.Snapshot.Snapshot do |> HTTP.delete([], _make_httpoison_options(options)) end - @doc false - @spec make_path(String.t(), [tuple()]) :: String.t() - def make_path(repo_name, snapshot_name, query_params \\ []) do - path = _make_base_path(repo_name, snapshot_name) - - case query_params do - [] -> path - _ -> _add_query_params(path, query_params) - end + This can also be used to stop currently running snapshot and restore operations. + """ + @spec delete(binary, binary, binary) :: HTTP.resp + def delete(elastic_url, repo, snapshot, options \\ []) do + url = HTTP.make_url(elastic_url, make_path(repo, snapshot)) + HTTP.delete(url, [], Keyword.get(options, :httpoison_options, [])) end - defp _make_httpoison_options(options), do: Keyword.get(options, :httpoison_options, []) - - defp _make_base_path(nil, nil), do: "/_snapshot" - defp _make_base_path(repo_name, nil), do: "/_snapshot/#{repo_name}" - - defp _make_base_path(repo_name, snapshot_name), - do: "/_snapshot/#{repo_name}/#{snapshot_name}" - - defp _add_query_params(path, query_params) do - query_string = - query_params - |> Enum.map_join("&", fn param -> - "#{elem(param, 0)}=#{elem(param, 1)}" - end) - - "#{path}?#{query_string}" - end + @doc false + @spec make_path(binary | nil, binary | nil) :: binary + def make_path(repo, snapshot) + def make_path(nil, nil), do: "/_snapshot" + def make_path(repo, nil), do: "/_snapshot/#{repo}" + def make_path(repo, snapshot), do: "/_snapshot/#{repo}/#{snapshot}" end diff --git a/lib/elastix/template.ex b/lib/elastix/template.ex new file mode 100644 index 0000000..6484e66 --- /dev/null +++ b/lib/elastix/template.ex @@ -0,0 +1,131 @@ +defmodule Elastix.Template do + @moduledoc """ + Index templates define settings and mappings that you can automatically apply + when creating new indices. Elasticsearch applies templates to new indices + based on an index pattern that matches the index name. + + [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) + """ + alias Elastix.{HTTP, JSON} + + @doc """ + Create a new index template. + + ## Examples + + iex> template = %{ + "index_patterns" => "logstash-*", + "mappings" => %{ + "dynamic_templates" => [ + %{ + "message_field" => %{ + "mapping" => %{"norms" => false, "type" => "text"}, + "match_mapping_type" => "string", + "path_match" => "message" + } + }, + %{ + "string_fields" => %{ + "mapping" => %{ + "fields" => %{ + "keyword" => %{"ignore_above" => 256, "type" => "keyword"} + }, + "norms" => false, + "type" => "text" + }, + "match" => "*", + "match_mapping_type" => "string" + } + } + ], + "properties" => %{ + "@timestamp" => %{"type" => "date"}, + "@version" => %{"type" => "keyword"}, + "geoip" => %{ + "dynamic" => true, + "properties" => %{ + "ip" => %{"type" => "ip"}, + "latitude" => %{"type" => "half_float"}, + "location" => %{"type" => "geo_point"}, + "longitude" => %{"type" => "half_float"} + } + } + } + }, + "settings" => %{"index.refresh_interval" => "5s", "number_of_shards" => 1}, + "version" => 60001 + } + iex> Elastix.Template.put("http://localhost:9200", "logstash", template) + {:ok, %HTTPoison.Response{...}} + """ + @spec put(binary, binary, binary | term, Keyword.t) :: HTTP.resp + def put(elastic_url, template, data, query_params \\ []) + def put(elastic_url, template, data, query_params) when is_binary(data) do + url = HTTP.make_url(elastic_url, make_path(template), query_params) + HTTP.put(url, data) + end + def put(elastic_url, template, data, query_params) do + put(elastic_url, template, JSON.encode!(data), query_params) + end + + @doc """ + Determine if an index template has alreay been registered. + + ## Examples + + iex> Elastix.Template.exists?("http://localhost:9200", "logstash") + {:ok, false} + + iex> Elastix.Template.exists?("http://localhost:9200", "logstash") + {:error, %HTTPoison.Error{id: nil, reason: :econnrefused}} + """ + @spec exists?(binary, binary, Keyword.t) :: {:ok, boolean} | {:error, HTTPoison.Error.t} + def exists?(elastic_url, template, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_path(template), query_params) + + case HTTP.head(url) do + {:ok, %{status_code: code}} when code >= 200 and code <= 299 -> + {:ok, true} + {:ok, _} -> + {:ok, false} + {:error, reason} -> + {:error, reason} + end + end + + @doc """ + Get info on an index template. + + ## Examples + + iex> Elastix.Template.get("http://localhost:9200", "logstash") + {:ok, %HTTPoison.Response{...}} + """ + @spec get(binary, binary, Keyword.t) :: HTTP.resp + def get(elastic_url, template, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_path(template), query_params) + HTTP.get(url) + end + + @doc """ + Delete an index template. + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-template.html) + # + ## Examples + + iex> Elastix.Template.get("http://localhost:9200", "logstash") + {:ok, %HTTPoison.Response{...}} + """ + @spec delete(binary, binary, Keyword.t) :: HTTP.resp + def delete(elastic_url, template, query_params \\ []) do + url = HTTP.make_url(elastic_url, make_path(template), query_params) + HTTP.delete(url) + end + + @doc false + # Convert params into path + @spec make_path(binary) :: binary + def make_path(template), do: "/_template/#{template}" + +end diff --git a/mix.exs b/mix.exs index 29c8c79..42d4e4a 100644 --- a/mix.exs +++ b/mix.exs @@ -29,6 +29,7 @@ defmodule Elastix.Mixfile do [ {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, {:credo, "~> 0.6", only: [:dev, :test]}, + {:dialyxir, "~> 0.5", only: [:dev, :test], runtime: false}, {:mix_test_watch, "~> 0.3", only: [:test, :dev]}, {:poison, "~> 3.0 or ~> 4.0", optional: true}, {:httpoison, "~> 1.4"}, diff --git a/mix.lock b/mix.lock index 26207b9..afbcf78 100644 --- a/mix.lock +++ b/mix.lock @@ -1,13 +1,13 @@ %{ - "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, - "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "01d479edba0569a7b7a2c8bf923feeb6dc6a358edc2965ef69aea9ba288bb243"}, - "credo": {:hex, :credo, "0.9.0", "5d1b494e4f2dc672b8318e027bd833dda69be71eaac6eedd994678be74ef7cb4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "61de62970f70111434b84ec272cc969ac693af1f3e112f23f1ef055e57e838e1"}, - "earmark": {:hex, :earmark, "1.2.4", "99b637c62a4d65a20a9fb674b8cffb8baa771c04605a80c911c4418c69b75439", [:mix], [], "hexpm", "1b34655872366414f69dd987cb121c049f76984b6ac69f52fff6d8fd64d29cfd"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, - "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, - "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm", "9a00246e8af58cdf465ae7c48fd6fd7ba2e43300413dfcc25447ecd3bf76f0c1"}, - "hackney": {:hex, :hackney, "1.14.3", "b5f6f5dcc4f1fba340762738759209e21914516df6be440d85772542d4a5e412", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "ed15491f324aa0e95647dca8ef4340418dac479d1204d57e455d52dcfba3f705"}, - "httpoison": {:hex, :httpoison, "1.4.0", "e0b3c2ad6fa573134e42194d13e925acfa8f89d138bc621ffb7b1989e6d22e73", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "37b6f39cb92136ee72276f4bf4da81495e7935d720c3a15daf1553953153e3f7"}, + "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], []}, + "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, + "credo": {:hex, :credo, "0.9.0", "5d1b494e4f2dc672b8318e027bd833dda69be71eaac6eedd994678be74ef7cb4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, optional: false]}]}, + "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"}, + "earmark": {:hex, :earmark, "1.2.4", "99b637c62a4d65a20a9fb674b8cffb8baa771c04605a80c911c4418c69b75439", [:mix], []}, + "ex_doc": {:hex, :ex_doc, "0.18.3", "f4b0e4a2ec6f333dccf761838a4b253d75e11f714b85ae271c9ae361367897b7", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]}, + "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []}, + "hackney": {:hex, :hackney, "1.14.3", "b5f6f5dcc4f1fba340762738759209e21914516df6be440d85772542d4a5e412", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "httpoison": {:hex, :httpoison, "1.4.0", "e0b3c2ad6fa573134e42194d13e925acfa8f89d138bc621ffb7b1989e6d22e73", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "httpotion": {:hex, :httpotion, "2.1.0"}, "ibrowse": {:git, "git://github.com/cmullaparthi/ibrowse.git", "d2e369ff42666c3574b8b7ec26f69027895c4d94", [tag: "v4.1.1"]}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, diff --git a/test/elastix/alias_test.exs b/test/elastix/alias_test.exs index 24bb5bb..572bd71 100644 --- a/test/elastix/alias_test.exs +++ b/test/elastix/alias_test.exs @@ -17,23 +17,23 @@ defmodule Elastix.AliasTest do test "aliases actions on existing index should respond with 200" do assert {:ok, %{status_code: _}} = Index.create(@test_url, @test_index, %{}) - assert {:ok, %{status_code: 200}} = - Alias.post(@test_url, [ - %{add: %{index: @test_index, alias: "alias1"}}, - %{remove: %{index: @test_index, alias: "alias1"}} - ]) + {:ok, response} = Alias.post(@test_url, [ + %{add: %{index: @test_index, alias: "alias1"}}, + %{remove: %{index: @test_index, alias: "alias1"}} + ]) + assert response.status_code == 200 end test "remove unkown alias on existing index should respond with 404" do assert {:ok, %{status_code: _}} = Index.create(@test_url, @test_index, %{}) - assert {:ok, %{status_code: 404}} = - Alias.post(@test_url, [%{remove: %{index: @test_index, alias: "alias1"}}]) + {:ok, response} = Alias.post(@test_url, [%{remove: %{index: @test_index, alias: "alias1"}}]) + assert response.status_code == 404 end test "alias actions on unknown index should respond with 404" do - assert {:ok, %{status_code: 404}} = - Alias.post(@test_url, [%{add: %{index: @test_index, alias: "alias1"}}]) + {:ok, response} = Alias.post(@test_url, [%{add: %{index: @test_index, alias: "alias1"}}]) + assert response.status_code == 404 end test "get all alias should respond with 200" do diff --git a/test/elastix/bulk_test.exs b/test/elastix/bulk_test.exs index a7bd875..f0b19a1 100644 --- a/test/elastix/bulk_test.exs +++ b/test/elastix/bulk_test.exs @@ -4,6 +4,8 @@ defmodule Elastix.BulkTest do alias Elastix.Bulk alias Elastix.Document + import ExUnit.CaptureLog + @test_url Elastix.config(:test_url) @test_index Elastix.config(:test_index) @@ -13,18 +15,18 @@ defmodule Elastix.BulkTest do :ok end - test "make_path should make url from index name, type, and query params" do - assert Bulk.make_path(@test_index, "tweet", version: 34, ttl: "1d") == - "/#{@test_index}/tweet/_bulk?version=34&ttl=1d" - end + describe "make_path/2" do + test "makes path from index name and type" do + assert Bulk.make_path(@test_index, "tweet") == "/#{@test_index}/tweet/_bulk" + end - test "make_path should make url from index name and query params" do - assert Bulk.make_path(@test_index, nil, version: 34, ttl: "1d") == - "/#{@test_index}/_bulk?version=34&ttl=1d" - end + test "makes path from index name" do + assert Bulk.make_path(@test_index, nil) == "/#{@test_index}/_bulk" + end - test "make_path should make url from query params" do - assert Bulk.make_path(nil, nil, version: 34, ttl: "1d") == "/_bulk?version=34&ttl=1d" + test "makes path with default" do + assert Bulk.make_path(nil, nil) == "/_bulk" + end end test "bulk accepts httpoison options" do @@ -49,47 +51,37 @@ defmodule Elastix.BulkTest do ]} end - test "post bulk should execute it", %{lines: lines} do + test "post bulk", %{lines: lines} do {:ok, response} = Bulk.post(@test_url, lines, index: @test_index, type: "message") - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end - test "post bulk with raw body should execute it", %{lines: lines} do - {:ok, response} = - Bulk.post_raw( - @test_url, - Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end), - index: @test_index, - type: "message" - ) - + test "post bulk with raw body", %{lines: lines} do + encoded = Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end) + {:ok, response} = Bulk.post_raw(@test_url, encoded, index: @test_index, type: "message") assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") + end - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + test "post_to_iolist/4 should log deprecation warning", %{lines: lines} do + assert capture_log(fn -> + Bulk.post_to_iolist(@test_url, lines, index: @test_index, type: "message") + end) =~ "This function is deprecated and will be removed in future releases; use Elastix.Bulk.post/4 instead" end + @tag capture_log: true test "post bulk sending iolist should execute it", %{lines: lines} do {:ok, response} = Bulk.post_to_iolist(@test_url, lines, index: @test_index, type: "message") - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end end @@ -106,43 +98,28 @@ defmodule Elastix.BulkTest do test "post bulk should execute it", %{lines: lines} do {:ok, response} = Bulk.post(@test_url, lines, index: @test_index) - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end test "post bulk with raw body should execute it", %{lines: lines} do - {:ok, response} = - Bulk.post_raw( - @test_url, - Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end), - index: @test_index - ) - + encoded = Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end) + {:ok, response} = Bulk.post_raw(@test_url, encoded, index: @test_index) assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end + @tag capture_log: true test "post bulk sending iolist should execute it", %{lines: lines} do {:ok, response} = Bulk.post_to_iolist(@test_url, lines, index: @test_index) - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end end @@ -159,42 +136,28 @@ defmodule Elastix.BulkTest do test "post bulk should execute it", %{lines: lines} do {:ok, response} = Bulk.post(@test_url, lines) - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end test "post bulk with raw body should execute it", %{lines: lines} do - {:ok, response} = - Bulk.post_raw( - @test_url, - Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end) - ) - + encoded = Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end) + {:ok, response} = Bulk.post_raw(@test_url, encoded) assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end + @tag capture_log: true test "post bulk sending iolist should execute it", %{lines: lines} do {:ok, response} = Bulk.post_to_iolist(@test_url, lines) - assert response.status_code == 200 - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "1") - - assert {:ok, %{status_code: 200}} = - Document.get(@test_url, @test_index, "message", "2") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "1") + assert {:ok, %{status_code: 200}} = Document.get(@test_url, @test_index, "message", "2") end end end diff --git a/test/elastix/document_test.exs b/test/elastix/document_test.exs index 3c5c614..91d8543 100644 --- a/test/elastix/document_test.exs +++ b/test/elastix/document_test.exs @@ -2,6 +2,7 @@ defmodule Elastix.DocumentTest do require Logger use ExUnit.Case alias Elastix.{Document, Index, Search} + alias Elastix.HTTP @test_url Elastix.config(:test_url) @test_index Elastix.config(:test_index) @@ -11,168 +12,199 @@ defmodule Elastix.DocumentTest do message: "trying out Elasticsearch" } + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = HTTP.get(@test_url) + {version, _rest} = Float.parse(response.body["version"]["number"]) + + {:ok, version: version} + end + + setup do Index.delete(@test_url, @test_index) :ok end - test "make_path should make url from index name, type, query params, id, and suffix" do - assert Document.make_path( - @test_index, - "tweet", - [version: 34, ttl: "1d"], - 2, - "_update" - ) == "/#{@test_index}/tweet/2/_update?version=34&ttl=1d" + describe "make_path/5" do + test "makes path from index name, type, query params, id, and suffix" do + assert Document.make_path_old(@test_index, "tweet", [version: 34, ttl: "1d"], 2, "_update") == + "/#{@test_index}/tweet/2/_update?version=34&ttl=1d" + end + + test "makes path from index name, type, and query params" do + assert Document.make_path_old(@test_index, "tweet", version: 34, ttl: "1d") == + "/#{@test_index}/tweet?version=34&ttl=1d" + end end - test "make_path without and id should make url from index name, type, and query params" do - assert Document.make_path(@test_index, "tweet", version: 34, ttl: "1d") == - "/#{@test_index}/tweet?version=34&ttl=1d" + describe "make_path/2" do + test "handles all options" do + assert Document.make_path(@test_index, %{id: 42}) == "/#{@test_index}/_doc/42" + assert Document.make_path(@test_index) == "/#{@test_index}/_doc/" + end end - test "index should create and index with data" do - {:ok, response} = Document.index(@test_url, @test_index, "message", 1, @data) + describe "index/6" do + test "old API indexes document", %{version: version} do + {:ok, response} = Document.index(@test_url, @test_index, "message", 1, @data) + + assert response.status_code == 201 + assert response.body["_id"] == "1" + assert response.body["_index"] == @test_index + assert response.body["_type"] == "message" + if version >= 6.0 do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end + end + + test "new API indexes document" do + {:ok, response} = Document.index(@test_url, @test_index, @data, %{id: 1}) + assert response.status_code == 201 + assert response.body["_id"] == "1" + assert response.body["_index"] == @test_index + assert response.body["result"] == "created" + end - assert response.status_code == 201 - assert response.body["_id"] == "1" - assert response.body["_index"] == @test_index - assert response.body["_type"] == "message" - assert response.body["created"] == true end - test "index_new should index data without an id" do + test "index_new should index data without an id", %{version: version} do {:ok, response} = Document.index_new(@test_url, @test_index, "message", @data) assert response.status_code == 201 assert response.body["_id"] assert response.body["_index"] == @test_index assert response.body["_type"] == "message" - assert response.body["created"] == true + if version >= 6.0 do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end end - test "get should return 404 if not index was created" do - {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + describe "get/5 old API" do + test "get returns 404 on unknown index" do + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 404 + end - assert response.status_code == 404 - end - - test "get should return data with 200 after index" do - Document.index(@test_url, @test_index, "message", 1, @data) - {:ok, response} = Document.get(@test_url, @test_index, "message", 1) - body = response.body + test "get returns data with 200 after index" do + Document.index(@test_url, @test_index, "message", 1, @data) + {:ok, %{status_code: 200, body: body}} = Document.get(@test_url, @test_index, "message", 1) - assert response.status_code == 200 - assert body["_source"]["user"] == "örelbörel" - assert body["_source"]["post_date"] == "2009-11-15T14:12:12" - assert body["_source"]["message"] == "trying out Elasticsearch" + assert body["_source"]["user"] == "örelbörel" + assert body["_source"]["post_date"] == "2009-11-15T14:12:12" + assert body["_source"]["message"] == "trying out Elasticsearch" + end end - test "delete should delete created index" do - Document.index(@test_url, @test_index, "message", 1, @data) - - {:ok, response} = Document.get(@test_url, @test_index, "message", 1) - assert response.status_code == 200 + describe "get/5 new API" do + test "get returns 404 on unknown index" do + {:ok, response} = Document.get(@test_url, @test_index, 1) + assert response.status_code == 404 + end - {:ok, response} = Document.delete(@test_url, @test_index, "message", 1) - assert response.status_code == 200 + test "get returns data with 200 after index" do + {:ok, %{status_code: 201}} = Document.index(@test_url, @test_index, @data, %{id: 1}) + {:ok, %{status_code: 200, body: body}} = Document.get(@test_url, @test_index, 1) - {:ok, response} = Document.get(@test_url, @test_index, "message", 1) - assert response.status_code == 404 + assert body["_source"]["user"] == "örelbörel" + assert body["_source"]["post_date"] == "2009-11-15T14:12:12" + assert body["_source"]["message"] == "trying out Elasticsearch" + end end - test "delete by query should remove all docs that match" do - Document.index(@test_url, @test_index, "message", 1, @data, refresh: true) - Document.index(@test_url, @test_index, "message", 2, @data, refresh: true) + describe "delete/5 old API" do - no_match = Map.put(@data, :user, "no match") - Document.index(@test_url, @test_index, "message", 3, no_match, refresh: true) + test "delete should delete created index" do + Document.index(@test_url, @test_index, "message", 1, @data) - match_all_query = %{"query" => %{"match_all" => %{}}} + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 200 - {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) - assert response.status_code == 200 - assert response.body["hits"]["total"] == 3 + {:ok, response} = Document.delete(@test_url, @test_index, "message", 1) + assert response.status_code == 200 - query = %{"query" => %{"match" => %{"user" => "örelbörel"}}} + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 404 + end - {:ok, response} = - Document.delete_matching(@test_url, @test_index, query, refresh: true) + test "delete by query should remove all docs that match" do + Document.index(@test_url, @test_index, "message", 1, @data, refresh: true) + Document.index(@test_url, @test_index, "message", 2, @data, refresh: true) - assert response.status_code == 200 + no_match = Map.put(@data, :user, "no match") + Document.index(@test_url, @test_index, "message", 3, no_match, refresh: true) - {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) - assert response.status_code == 200 - assert response.body["hits"]["total"] == 1 - end + match_all_query = %{"query" => %{"match_all" => %{}}} - test "update can partially update document" do - Document.index(@test_url, @test_index, "message", 1, @data) + {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 3 - new_post_date = "2017-03-17T14:12:12" - patch = %{doc: %{post_date: new_post_date}} + query = %{"query" => %{"match" => %{"user" => "örelbörel"}}} - {:ok, response} = Document.update(@test_url, @test_index, "message", 1, patch) - assert response.status_code == 200 + {:ok, response} = + Document.delete_matching(@test_url, @test_index, query, refresh: true) - {:ok, %{body: body, status_code: status_code}} = - Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 200 - assert status_code == 200 - assert body["_source"]["user"] == "örelbörel" - assert body["_source"]["post_date"] == new_post_date - assert body["_source"]["message"] == "trying out Elasticsearch" + {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 1 + end end - test "update by query can update a list of docs by matching query" do - Document.index(@test_url, @test_index, "message", 1, @data, refresh: true) - Document.index(@test_url, @test_index, "message", 2, @data, refresh: true) + describe "delete/5 new API" do - new_post_date = "2020-06-03T14:12:12" + test "deletes created index" do + Document.index(@test_url, @test_index, @data, %{id: 1}) - script = %{ - inline: "ctx._source.post_date = '#{new_post_date}'", - lang: "painless" - } + {:ok, response} = Document.get(@test_url, @test_index, 1) + assert response.status_code == 200 - query = %{"term" => %{"user" => "örelbörel"}} + {:ok, response} = Document.delete(@test_url, @test_index, 1) + assert response.status_code == 200 - {:ok, response} = - Document.update_by_query(@test_url, @test_index, query, script, refresh: true) + {:ok, response} = Document.get(@test_url, @test_index, 1) + assert response.status_code == 404 + end - assert response.status_code == 200 + test "delete by query should remove all docs that match" do + Document.index(@test_url, @test_index, @data, %{id: 1}, refresh: true) + Document.index(@test_url, @test_index, @data, %{id: 2}, refresh: true) - {:ok, %{body: body, status_code: status_code}} = - Document.get(@test_url, @test_index, "message", 1) + no_match = Map.put(@data, :user, "no match") + Document.index(@test_url, @test_index, no_match, %{id: 3}, refresh: true) - assert status_code == 200 - assert body["_source"]["user"] == "örelbörel" - assert body["_source"]["post_date"] == new_post_date - assert body["_source"]["message"] == "trying out Elasticsearch" + match_all_query = %{"query" => %{"match_all" => %{}}} - {:ok, %{body: body, status_code: status_code}} = - Document.get(@test_url, @test_index, "message", 2) + {:ok, response} = Search.search(@test_url, @test_index, [], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 3 - assert status_code == 200 - assert body["_source"]["user"] == "örelbörel" - assert body["_source"]["post_date"] == new_post_date - assert body["_source"]["message"] == "trying out Elasticsearch" - end + query = %{"query" => %{"match" => %{"user" => "örelbörel"}}} - test "update by query doesnt update when no matches are found" do - Document.index(@test_url, @test_index, "message", 1, @data, refresh: true) + {:ok, response} = Document.delete_matching(@test_url, @test_index, query, refresh: true) + assert response.status_code == 200 - script = %{ - inline: "ctx._source.message = 'updated message'", - lang: "painless" - } + {:ok, response} = Search.search(@test_url, @test_index, [], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 1 + end + end - query = %{"term" => %{"user" => "other user"}} + test "update can partially update document" do + Document.index(@test_url, @test_index, "message", 1, @data) - {:ok, response} = - Document.update_by_query(@test_url, @test_index, query, script, refresh: true) + new_post_date = "2017-03-17T14:12:12" + patch = %{doc: %{post_date: new_post_date}} + {:ok, response} = Document.update(@test_url, @test_index, "message", 1, patch) assert response.status_code == 200 {:ok, %{body: body, status_code: status_code}} = @@ -180,78 +212,118 @@ defmodule Elastix.DocumentTest do assert status_code == 200 assert body["_source"]["user"] == "örelbörel" - assert body["_source"]["post_date"] == "2009-11-15T14:12:12" + assert body["_source"]["post_date"] == new_post_date assert body["_source"]["message"] == "trying out Elasticsearch" end - test "can get multiple documents (multi get)" do - Document.index(@test_url, @test_index, "message", 1, @data) - Document.index(@test_url, @test_index, "message", 2, @data) - - query = %{ - "docs" => [ - %{ - "_index" => @test_index, - "_type" => "message", - "_id" => "1" - }, - %{ - "_index" => @test_index, - "_type" => "message", - "_id" => "2" - } - ] - } - - {:ok, %{body: body, status_code: status_code}} = Document.mget(@test_url, query) - - assert status_code === 200 - assert length(body["docs"]) == 2 + describe "mget/5 old API" do + + test "can get multiple documents (multi get)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{"_index" => @test_index, "_type" => "message", "_id" => "1"}, + %{"_index" => @test_index, + "_type" => "message", + "_id" => "2" + } + ] + } + + {:ok, %{body: body, status_code: status_code}} = Document.mget(@test_url, query) + + assert status_code === 200 + assert length(body["docs"]) == 2 + end + + test "can get multiple documents (multi get with index)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{"_type" => "message", "_id" => "1"}, + %{"_type" => "message", "_id" => "2"} + ] + } + + {:ok, %{body: body, status_code: status_code}} = + Document.mget(@test_url, query, @test_index) + + assert status_code === 200 + assert length(body["docs"]) == 2 + end + + test "can get multiple documents (multi get with index and type)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{"_id" => "1"}, + %{"_id" => "2"} + ] + } + + {:ok, %{body: body, status_code: status_code}} = + Document.mget(@test_url, query, @test_index, "message") + + assert status_code === 200 + assert length(body["docs"]) == 2 + end end - test "can get multiple documents (multi get with index)" do - Document.index(@test_url, @test_index, "message", 1, @data) - Document.index(@test_url, @test_index, "message", 2, @data) - - query = %{ - "docs" => [ - %{ - "_type" => "message", - "_id" => "1" - }, - %{ - "_type" => "message", - "_id" => "2" - } - ] - } - - {:ok, %{body: body, status_code: status_code}} = - Document.mget(@test_url, query, @test_index) - - assert status_code === 200 - assert length(body["docs"]) == 2 + describe "mget/5 new API" do + test "can get multiple documents (multi get)" do + Document.index(@test_url, @test_index, @data, %{id: 1}) + Document.index(@test_url, @test_index, @data, %{id: 2}) + + query = %{ + "docs" => [ + %{"_index" => @test_index, "_id" => "1"}, + %{"_index" => @test_index, "_id" => "2"} + ] + } + + {:ok, %{body: body, status_code: status_code}} = Document.mget(@test_url, query) + + assert status_code === 200 + assert length(body["docs"]) == 2 + end + + test "can get multiple documents (multi get with index)" do + Document.index(@test_url, @test_index, @data, %{id: 1}) + Document.index(@test_url, @test_index, @data, %{id: 2}) + + query = %{ + "docs" => [ + %{"_id" => "1"}, + %{"_id" => "2"} + ] + } + + {:ok, response} = Document.mget(@test_url, query, @test_index) + assert response.status_code === 200 + assert length(response.body["docs"]) == 2 + end + + test "can get multiple documents" do + Document.index(@test_url, @test_index, @data, %{id: 1}) + Document.index(@test_url, @test_index, @data, %{id: 2}) + + query = %{ + "docs" => [ + %{"_id" => "1"}, + %{"_id" => "2"} + ] + } + + {:ok, response} = Document.mget(@test_url, query, @test_index) + assert response.status_code == 200 + assert length(response.body["docs"]) == 2 + end end - test "can get multiple documents (multi get with index and type)" do - Document.index(@test_url, @test_index, "message", 1, @data) - Document.index(@test_url, @test_index, "message", 2, @data) - - query = %{ - "docs" => [ - %{ - "_id" => "1" - }, - %{ - "_id" => "2" - } - ] - } - - {:ok, %{body: body, status_code: status_code}} = - Document.mget(@test_url, query, @test_index, "message") - - assert status_code === 200 - assert length(body["docs"]) == 2 - end end diff --git a/test/elastix/http_test.exs b/test/elastix/http_test.exs index 57b95a2..fbe2bc2 100644 --- a/test/elastix/http_test.exs +++ b/test/elastix/http_test.exs @@ -1,32 +1,60 @@ defmodule Elastix.HTTPTest do + @app :elastix + use ExUnit.Case + import ExUnit.CaptureLog + alias Elastix.HTTP - @test_url Elastix.config(:test_url) + @test_url Application.get_env(@app, :test_url) - test "prepare_url/2 should concat url with path" do - assert HTTP.prepare_url("http://127.0.0.1:9200/", "/some_path") == - "http://127.0.0.1:9200/some_path" - end + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = HTTP.get(@test_url) + {version, _rest} = Float.parse(response.body["version"]["number"]) - test "prepare_url/2 should concat url with a list of path parts" do - assert HTTP.prepare_url("http://127.0.0.1:9200/", ["/some/", "/path/"]) == - "http://127.0.0.1:9200/some/path" + {:ok, version: version} end test "get should respond with 200" do - {_, response} = HTTP.get(@test_url, []) + {:ok, response} = HTTP.get(@test_url, []) assert response.status_code == 200 end - test "post should respond with 400" do + test "make_url/3 handles all options" do + url = "http://localhost:9200" + assert HTTP.make_url(url, "foo") == "#{url}/foo" + assert HTTP.make_url(url, ["foo", "bar"]) == "#{url}/foo/bar" + assert HTTP.make_url(url, "/foo") == "#{url}/foo" + assert HTTP.make_url(url, "/foo", this: true) == "#{url}/foo?this=true" + assert HTTP.make_url(url, "/foo", %{biz: :baz, hello: 1}) == "#{url}/foo?biz=baz&hello=1" + end + + test "add_content_type_header/1" do + default_headers = [{"Content-Type", "application/json; charset=UTF-8"}] + bulk_headers = [{"Content-Type", "application/x-ndjson"}] + + assert HTTP.add_content_type_header([]) == default_headers + assert HTTP.add_content_type_header(bulk_headers) == bulk_headers + end + + test "post should respond with 400", %{version: version} do {_, response} = HTTP.post(@test_url, []) - assert response.status_code == 400 + if version >= 6.0 do + assert response.status_code == 405 + else + assert response.status_code == 400 + end end - test "put should respond with 400" do + test "put should respond with 400", %{version: version} do {_, response} = HTTP.put(@test_url, []) - assert response.status_code == 400 + if version >= 6.0 do + assert response.status_code == 405 + else + assert response.status_code == 400 + end end test "delete should respond with 400" do @@ -44,11 +72,34 @@ defmodule Elastix.HTTPTest do assert HTTP.process_response_body(body) == body end - test "process_response_body parsed the body into an atom key map if configured" do - body = "{\"some\":\"json\"}" - Application.put_env(:elastix, :poison_options, keys: :atoms) - assert HTTP.process_response_body(body) == %{some: "json"} - Application.delete_env(:elastix, :poison_options) + describe "Environment specific tests" do + setup do + Application.put_env(:elastix, :poison_options, keys: :atoms) + Application.put_env(:elastix, :json_options, keys: :atoms) + + on_exit(fn -> + Application.delete_env(:elastix, :poison_options) + Application.delete_env(:elastix, :json_options) + end) + + :ok + end + + test "using :poison_options logs a deprecation warning" do + assert capture_log(fn -> + body = "{\"some\":\"json\"}" + Application.put_env(:elastix, :poison_options, keys: :atoms) + %{some: "json"} = HTTP.process_response_body(body) + end) =~ "Using :poison_options is deprecated and might not work in future releases; use :json_options instead" + end + + @tag capture_log: true + test "process_response_body parsed the body into an atom key map if configured" do + body = "{\"some\":\"json\"}" + Application.put_env(:elastix, :json_options, keys: :atoms) + assert HTTP.process_response_body(body) == %{some: "json"} + Application.delete_env(:elastix, :poison_options) + end end test "adding custom headers" do @@ -70,6 +121,17 @@ defmodule Elastix.HTTPTest do assert {"Content-Type", "application/json; charset=UTF-8"} in fake_resp end + test "add_query_params/2" do + params = %{foo: "bar", biz: 1} + base = "http://localhost:9200/base" + + assert HTTP.add_query_params(base, nil) == base + assert HTTP.add_query_params(base, %{}) == base + assert HTTP.add_query_params(base, []) == base + assert HTTP.add_query_params(base, foo: :bar, biz: 1) == "#{base}?foo=bar&biz=1" + assert HTTP.add_query_params(base, params) == "#{base}?biz=1&foo=bar" + end + # Test implementation of custom headers def add_custom_headers(request, :foo) do [{"test", "pass"} | request.headers] @@ -79,4 +141,5 @@ defmodule Elastix.HTTPTest do def return_headers(_, %HTTPoison.Request{headers: headers}, _, _, _, _) do headers end + end diff --git a/test/elastix/mapping_test.exs b/test/elastix/mapping_test.exs index 0699a3d..fb1c4d1 100644 --- a/test/elastix/mapping_test.exs +++ b/test/elastix/mapping_test.exs @@ -3,6 +3,7 @@ defmodule Elastix.MappingTest do alias Elastix.Index alias Elastix.Mapping alias Elastix.Document + alias Elastix.HTTP @test_url Elastix.config(:test_url) @test_index Elastix.config(:test_index) @@ -24,6 +25,16 @@ defmodule Elastix.MappingTest do message: true } + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = HTTP.get(@test_url) + version_string = response.body["version"]["number"] + version = Elastix.version_to_tuple(version_string) + + {:ok, version: version} + end + setup do Index.delete(@test_url, @test_index) Index.delete(@test_url, @test_index2) @@ -31,26 +42,37 @@ defmodule Elastix.MappingTest do :ok end - defp elasticsearch_version do - %HTTPoison.Response{body: %{"version" => %{"number" => v}}, status_code: 200} = - Elastix.HTTP.get!(@test_url) - - v |> String.split([".", "-"]) |> Enum.take(3) |> Enum.map(&String.to_integer/1) - |> List.to_tuple() - end - - test "make_path should make url from index names, types, and query params" do - assert Mapping.make_path([@test_index], ["tweet"], version: 34, ttl: "1d") == - "/#{@test_index}/_mapping/tweet?version=34&ttl=1d" + test "put_path/4 handles all combinations of params" do + # Old + assert Mapping.put_path("foo", "biz", %{}, []) == "/foo/_mapping/biz" + assert Mapping.put_path(["foo"], "biz", %{}, []) == "/foo/_mapping/biz" + assert Mapping.put_path(["foo", "bar"], "biz", %{}, []) == "/foo,bar/_mapping/biz" + assert Mapping.put_path(["foo", "bar"], "biz", %{}, version: 34, ttl: "1d") == + "/foo,bar/_mapping/biz?version=34&ttl=1d" + + assert Mapping.put_path("foo", %{}, [], []) == "/foo/_mapping" + assert Mapping.put_path("foo", %{}, [version: 34, ttl: "1d"], []) == + "/foo/_mapping?version=34&ttl=1d" end - test "make_all_path should make url from types, and query params" do - assert Mapping.make_all_path(["tweet"], version: 34, ttl: "1d") == - "/_mapping/tweet?version=34&ttl=1d" + test "get_path/3 handles all combinations of params" do + # Old + assert Mapping.get_path("foo", "biz", []) == "/foo/_mapping/biz" + assert Mapping.get_path(["foo"], "biz", []) == "/foo/_mapping/biz" + assert Mapping.get_path(["foo"], ["biz"], []) == "/foo/_mapping/biz" + assert Mapping.get_path("foo", ["biz"], []) == "/foo/_mapping/biz" + assert Mapping.get_path(["foo", "bar"], ["biz", "baz"], []) == "/foo,bar/_mapping/biz,baz" + assert Mapping.get_path(["foo", "bar"], ["biz", "baz"], version: 34, ttl: "1d") == + "/foo,bar/_mapping/biz,baz?version=34&ttl=1d" + + # New + assert Mapping.get_path("foo", [], []) == "/foo/_mapping" + assert Mapping.get_path("foo", [version: 34, ttl: "1d"], []) == + "/foo/_mapping?version=34&ttl=1d" end - test "make_all_path should make url from query params" do - assert Mapping.make_all_path(version: 34, ttl: "1d") == "/_mapping?version=34&ttl=1d" + test "make_all_path/2 makes url from types" do + assert Mapping.make_all_path(["tweet"]) == "/_mapping/tweet" end test "put mapping with no index should error" do @@ -73,17 +95,24 @@ defmodule Elastix.MappingTest do assert response.status_code == 404 end - test "get with non existing mapping" do + test "get with non existing mapping", %{version: version} do Index.create(@test_url, @test_index, %{}) - {:ok, response} = Mapping.get(@test_url, @test_index, "message") - if elasticsearch_version() >= {5, 5, 0} do - assert response.body["error"]["reason"] == "type[[message]] missing" - else - assert response.body == %{} + cond do + version >= {6, 0, 0} -> + {:ok, response} = Mapping.get(@test_url, @test_index, include_type_name: false) + assert response.body == %{@test_index => %{"mappings" => %{}}} + assert response.status_code == 200 + version >= {5, 5, 0} -> + {:ok, response} = Mapping.get(@test_url, @test_index, "message") + assert response.body["error"]["reason"] == "type[[message]] missing" + assert response.status_code == 404 + true -> + {:ok, response} = Mapping.get(@test_url, @test_index, "message") + assert response.body == %{} + assert response.status_code == 404 end - assert response.status_code == 404 end test "get mapping should return mapping" do @@ -95,15 +124,24 @@ defmodule Elastix.MappingTest do assert response.body[@test_index]["mappings"]["message"] == @target_mapping end - test "get mapping for several types should return several mappings" do + test "get mapping for several types should return several mappings", %{version: version} do Index.create(@test_url, @test_index, %{}) - Mapping.put(@test_url, @test_index, "message", @mapping) - Mapping.put(@test_url, @test_index, "comment", @mapping) - {:ok, response} = Mapping.get(@test_url, @test_index, ["message", "comment"]) - assert response.status_code == 200 - assert response.body[@test_index]["mappings"]["message"] == @target_mapping - assert response.body[@test_index]["mappings"]["comment"] == @target_mapping + if version >= {6, 0, 0} do + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index, "comment", @mapping) + + {:ok, response} = Mapping.get(@test_url, @test_index, ["message", "comment"]) + assert response.status_code == 404 + else + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index, "comment", @mapping) + + {:ok, response} = Mapping.get(@test_url, @test_index, ["message", "comment"]) + assert response.status_code == 200 + assert response.body[@test_index]["mappings"]["message"] == @target_mapping + assert response.body[@test_index]["mappings"]["comment"] == @target_mapping + end end test "get_all mappings should return mappings for all indexes and types" do @@ -130,7 +168,7 @@ defmodule Elastix.MappingTest do assert response.body[@test_index2]["mappings"]["comment"] == @target_mapping end - test "put document with mapping should put document" do + test "put document with mapping should put document", %{version: version} do Index.create(@test_url, @test_index, %{}) Mapping.put(@test_url, @test_index, "message", @mapping) @@ -140,6 +178,11 @@ defmodule Elastix.MappingTest do assert response.body["_id"] == "1" assert response.body["_index"] == @test_index assert response.body["_type"] == "message" - assert response.body["created"] == true + + if version >= {6, 0, 0} do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end end end diff --git a/test/elastix/old/alias_test.exs b/test/elastix/old/alias_test.exs new file mode 100644 index 0000000..56a34dd --- /dev/null +++ b/test/elastix/old/alias_test.exs @@ -0,0 +1,36 @@ +defmodule Elastix.Old.AliasTest do + use ExUnit.Case + alias Elastix.Alias + alias Elastix.Index + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + + setup do + Index.delete(@test_url, @test_index) + + :ok + end + + test "aliases actions on existing index should respond with 200" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + + assert {:ok, %{status_code: 200}} = + Alias.post(@test_url, [ + %{add: %{index: @test_index, alias: "alias1"}}, + %{remove: %{index: @test_index, alias: "alias1"}} + ]) + end + + test "remove unkown alias on existing index should respond with 404" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + + assert {:ok, %{status_code: 404}} = + Alias.post(@test_url, [%{remove: %{index: @test_index, alias: "alias1"}}]) + end + + test "alias actions on unknown index should respond with 404" do + assert {:ok, %{status_code: 404}} = + Alias.post(@test_url, [%{add: %{index: @test_index, alias: "alias1"}}]) + end +end diff --git a/test/elastix/old/bulk_test.exs b/test/elastix/old/bulk_test.exs new file mode 100644 index 0000000..0fa19eb --- /dev/null +++ b/test/elastix/old/bulk_test.exs @@ -0,0 +1,207 @@ +defmodule Elastix.Old.BulkTest do + use ExUnit.Case + alias Elastix.Index + alias Elastix.Bulk + alias Elastix.Document + @old false + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + + setup do + Index.delete(@test_url, @test_index) + + :ok + end + + test "make_path should make url from index name, type, and query params" do + if @old do + assert Bulk.make_path(@test_index, "tweet", version: 34, ttl: "1d") == + "/#{@test_index}/tweet/_bulk?version=34&ttl=1d" + end + end + + test "make_path should make url from index name and query params" do + if @old do + assert Bulk.make_path(@test_index, nil, version: 34, ttl: "1d") == + "/#{@test_index}/_bulk?version=34&ttl=1d" + end + end + + test "make_path should make url from query params" do + if @old do + assert Bulk.make_path(nil, nil, version: 34, ttl: "1d") == "/_bulk?version=34&ttl=1d" + end + end + + test "bulk accepts httpoison options" do + lines = [ + %{index: %{_id: "1"}}, + %{field: "value1"}, + %{index: %{_id: "2"}}, + %{field: "value2"} + ] + {:error, %HTTPoison.Error{reason: :timeout}} = + Bulk.post @test_url, lines, index: @test_index, type: "message", httpoison_options: [recv_timeout: 0] + end + + describe "test bulks with index and type in URL" do + setup do + {:ok, + lines: [ + %{index: %{_id: "1"}}, + %{field: "value1"}, + %{index: %{_id: "2"}}, + %{field: "value2"} + ]} + end + + test "post bulk should execute it", %{lines: lines} do + {:ok, response} = Bulk.post(@test_url, lines, index: @test_index, type: "message") + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk with raw body should execute it", %{lines: lines} do + {:ok, response} = + Bulk.post_raw( + @test_url, + Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end), + index: @test_index, + type: "message" + ) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk sending iolist should execute it", %{lines: lines} do + {:ok, response} = + Bulk.post_to_iolist(@test_url, lines, index: @test_index, type: "message") + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + end + + describe "test bulks with index only in URL" do + setup do + {:ok, + lines: [ + %{index: %{_id: "1", _type: "message"}}, + %{field: "value1"}, + %{index: %{_id: "2", _type: "message"}}, + %{field: "value2"} + ]} + end + + test "post bulk should execute it", %{lines: lines} do + {:ok, response} = Bulk.post(@test_url, lines, index: @test_index) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk with raw body should execute it", %{lines: lines} do + {:ok, response} = + Bulk.post_raw( + @test_url, + Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end), + index: @test_index + ) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk sending iolist should execute it", %{lines: lines} do + {:ok, response} = Bulk.post_to_iolist(@test_url, lines, index: @test_index) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + end + + describe "test bulks without index nor type in URL" do + setup do + {:ok, + lines: [ + %{index: %{_id: "1", _type: "message", _index: @test_index}}, + %{field: "value1"}, + %{index: %{_id: "2", _type: "message", _index: @test_index}}, + %{field: "value2"} + ]} + end + + test "post bulk should execute it", %{lines: lines} do + {:ok, response} = Bulk.post(@test_url, lines) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk with raw body should execute it", %{lines: lines} do + {:ok, response} = + Bulk.post_raw( + @test_url, + Enum.map(lines, fn line -> Poison.encode!(line) <> "\n" end) + ) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + + test "post bulk sending iolist should execute it", %{lines: lines} do + {:ok, response} = Bulk.post_to_iolist(@test_url, lines) + + assert response.status_code == 200 + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "1") + + assert {:ok, %{status_code: 200}} = + Document.get(@test_url, @test_index, "message", "2") + end + end +end diff --git a/test/elastix/old/document_test.exs b/test/elastix/old/document_test.exs new file mode 100644 index 0000000..7a0b783 --- /dev/null +++ b/test/elastix/old/document_test.exs @@ -0,0 +1,220 @@ +defmodule Elastix.Old.DocumentTest do + require Logger + use ExUnit.Case + alias Elastix.{Document, Index, Search} + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + @data %{ + user: "örelbörel", + post_date: "2009-11-15T14:12:12", + message: "trying out Elasticsearch" + } + @old false + + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = Elastix.HTTP.get(@test_url) + {version, _rest} = Float.parse(response.body["version"]["number"]) + + {:ok, version: version} + end + + setup do + Index.delete(@test_url, @test_index) + + :ok + end + + test "make_path should make url from index name, type, query params, id, and suffix" do + if @old do + assert Document.make_path( + @test_index, + "tweet", + [version: 34, ttl: "1d"], + 2, + "_update" + ) == "/#{@test_index}/tweet/2/_update?version=34&ttl=1d" + end + end + + test "make_path without and id should make url from index name, type, and query params" do + if @old do + assert Document.make_path(@test_index, "tweet", version: 34, ttl: "1d") == + "/#{@test_index}/tweet?version=34&ttl=1d" + end + end + + test "index should create and index with data", %{version: version} do + {:ok, response} = Document.index(@test_url, @test_index, "message", 1, @data) + + assert response.status_code == 201 + assert response.body["_id"] == "1" + assert response.body["_index"] == @test_index + assert response.body["_type"] == "message" + if version >= 6.0 do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end + end + + test "index_new should index data without an id", %{version: version} do + {:ok, response} = Document.index_new(@test_url, @test_index, "message", @data) + + assert response.status_code == 201 + assert response.body["_id"] + assert response.body["_index"] == @test_index + assert response.body["_type"] == "message" + if version >= 6.0 do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end + end + + test "get should return 404 if not index was created" do + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + + assert response.status_code == 404 + end + + test "get should return data with 200 after index" do + Document.index(@test_url, @test_index, "message", 1, @data) + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + body = response.body + + assert response.status_code == 200 + assert body["_source"]["user"] == "örelbörel" + assert body["_source"]["post_date"] == "2009-11-15T14:12:12" + assert body["_source"]["message"] == "trying out Elasticsearch" + end + + test "delete should delete created index" do + Document.index(@test_url, @test_index, "message", 1, @data) + + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 200 + + {:ok, response} = Document.delete(@test_url, @test_index, "message", 1) + assert response.status_code == 200 + + {:ok, response} = Document.get(@test_url, @test_index, "message", 1) + assert response.status_code == 404 + end + + test "delete by query should remove all docs that match" do + Document.index(@test_url, @test_index, "message", 1, @data, refresh: true) + Document.index(@test_url, @test_index, "message", 2, @data, refresh: true) + + no_match = Map.put(@data, :user, "no match") + Document.index(@test_url, @test_index, "message", 3, no_match, refresh: true) + + match_all_query = %{"query" => %{"match_all" => %{}}} + + {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 3 + + query = %{"query" => %{"match" => %{"user" => "örelbörel"}}} + + {:ok, response} = + Document.delete_matching(@test_url, @test_index, query, refresh: true) + + assert response.status_code == 200 + + {:ok, response} = Search.search(@test_url, @test_index, ["message"], match_all_query) + assert response.status_code == 200 + assert response.body["hits"]["total"] == 1 + end + + test "update can partially update document" do + Document.index(@test_url, @test_index, "message", 1, @data) + + new_post_date = "2017-03-17T14:12:12" + patch = %{doc: %{post_date: new_post_date}} + + {:ok, response} = Document.update(@test_url, @test_index, "message", 1, patch) + assert response.status_code == 200 + + {:ok, %{body: body, status_code: status_code}} = + Document.get(@test_url, @test_index, "message", 1) + + assert status_code == 200 + assert body["_source"]["user"] == "örelbörel" + assert body["_source"]["post_date"] == new_post_date + assert body["_source"]["message"] == "trying out Elasticsearch" + end + + test "can get multiple documents (multi get)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{ + "_index" => @test_index, + "_type" => "message", + "_id" => "1" + }, + %{ + "_index" => @test_index, + "_type" => "message", + "_id" => "2" + } + ] + } + + {:ok, %{body: body, status_code: status_code}} = Document.mget(@test_url, query) + + assert status_code === 200 + assert length(body["docs"]) == 2 + end + + test "can get multiple documents (multi get with index)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{ + "_type" => "message", + "_id" => "1" + }, + %{ + "_type" => "message", + "_id" => "2" + } + ] + } + + {:ok, %{body: body, status_code: status_code}} = + Document.mget(@test_url, query, @test_index) + + assert status_code === 200 + assert length(body["docs"]) == 2 + end + + test "can get multiple documents (multi get with index and type)" do + Document.index(@test_url, @test_index, "message", 1, @data) + Document.index(@test_url, @test_index, "message", 2, @data) + + query = %{ + "docs" => [ + %{ + "_id" => "1" + }, + %{ + "_id" => "2" + } + ] + } + + {:ok, %{body: body, status_code: status_code}} = + Document.mget(@test_url, query, @test_index, "message") + + assert status_code === 200 + assert length(body["docs"]) == 2 + end +end diff --git a/test/elastix/old/http_test.exs b/test/elastix/old/http_test.exs new file mode 100644 index 0000000..7117352 --- /dev/null +++ b/test/elastix/old/http_test.exs @@ -0,0 +1,104 @@ +defmodule Elastix.Old.HTTPTest do + use ExUnit.Case + alias Elastix.HTTP + @old false + + @test_url Elastix.config(:test_url) + + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = HTTP.get(@test_url) + {version, _rest} = Float.parse(response.body["version"]["number"]) + + {:ok, version: version} + end + + test "prepare_url/2 should concat url with path" do + if @old do + assert HTTP.prepare_url("http://127.0.0.1:9200/", "/some_path") == + "http://127.0.0.1:9200/some_path" + end + end + + test "prepare_url/2 should concat url with a list of path parts" do + if @old do + assert HTTP.prepare_url("http://127.0.0.1:9200/", ["/some/", "/path/"]) == + "http://127.0.0.1:9200/some/path" + end + end + + test "get should respond with 200" do + {_, response} = HTTP.get(@test_url, []) + assert response.status_code == 200 + end + + test "post should respond with 400", %{version: version} do + {_, response} = HTTP.post(@test_url, []) + if version >= 6.0 do + assert response.status_code == 405 + else + assert response.status_code == 400 + end + end + + test "put should respond with 400", %{version: version} do + {_, response} = HTTP.put(@test_url, []) + if version >= 6.0 do + assert response.status_code == 405 + else + assert response.status_code == 400 + end + end + + test "delete should respond with 400" do + {_, response} = HTTP.delete(@test_url, []) + assert response.status_code == 400 + end + + test "process_response_body should parse the json body into a map" do + body = "{\"some\":\"json\"}" + assert HTTP.process_response_body(body) == %{"some" => "json"} + end + + test "process_response_body returns the raw body if it cannot be parsed as json" do + body = "no_json" + assert HTTP.process_response_body(body) == body + end + + test "process_response_body parsed the body into an atom key map if configured" do + body = "{\"some\":\"json\"}" + Application.put_env(:elastix, :poison_options, keys: :atoms) + assert HTTP.process_response_body(body) == %{some: "json"} + Application.delete_env(:elastix, :poison_options) + end + + test "adding custom headers" do + Application.put_env( + :elastix, + :custom_headers, + {__MODULE__, :add_custom_headers, [:foo]} + ) + + Application.put_env(:elastix, :test_request_mfa, {__MODULE__, :return_headers, []}) + + fake_resp = + HTTP.request("GET", "#{@test_url}/_cluster/health", "", [{"yolo", "true"}]) + + Application.delete_env(:elastix, :test_request_mfa) + Application.delete_env(:elastix, :custom_headers) + assert {"yolo", "true"} in fake_resp + assert {"test", "pass"} in fake_resp + assert {"Content-Type", "application/json; charset=UTF-8"} in fake_resp + end + + # Test implementation of custom headers + def add_custom_headers(request, :foo) do + [{"test", "pass"} | request.headers] + end + + # Skip actual http request so we can test what we sent to poison. + def return_headers(_, %HTTPoison.Request{headers: headers}, _, _, _, _) do + headers + end +end diff --git a/test/elastix/old/index_test.exs b/test/elastix/old/index_test.exs new file mode 100644 index 0000000..85656fe --- /dev/null +++ b/test/elastix/old/index_test.exs @@ -0,0 +1,67 @@ +defmodule Elastix.Old.IndexTest do + use ExUnit.Case + alias Elastix.Index + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + + setup do + Index.delete(@test_url, @test_index) + + :ok + end + + test "exists? should return false if index is not created" do + assert {:ok, false} == Index.exists?(@test_url, @test_index) + end + + test "exists? should return true if index is created" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, true} == Index.exists?(@test_url, @test_index) + end + + test "create then delete should respond with 200" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 200}} = Index.delete(@test_url, @test_index) + end + + test "double create should respond with 400" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 400}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 200}} = Index.delete(@test_url, @test_index) + end + + test "get of uncreated index should respond with 404" do + assert {:ok, %{status_code: 404}} = Index.get(@test_url, @test_index) + end + + test "get of created index should respond with 200 and index data" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + + {:ok, index} = Index.get(@test_url, @test_index) + assert index.status_code == 200 + + assert index.body[@test_index] + end + + test "refresh of uncreated index should respond with 404" do + assert {:ok, %{status_code: 404}} = Index.refresh(@test_url, @test_index) + end + + test "refresh of existing index should respond with 200" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 200}} = Index.refresh(@test_url, @test_index) + end + + test "open of existing index should respond with 200" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 200}} = Index.refresh(@test_url, @test_index) + assert {:ok, %{status_code: 200}} = Index.open(@test_url, @test_index) + end + + test "close of existing index should respond with 200" do + assert {:ok, %{status_code: 200}} = Index.create(@test_url, @test_index, %{}) + assert {:ok, %{status_code: 200}} = Index.refresh(@test_url, @test_index) + assert {:ok, %{status_code: 200}} = Index.close(@test_url, @test_index) + end +end diff --git a/test/elastix/old/mapping_test.exs b/test/elastix/old/mapping_test.exs new file mode 100644 index 0000000..518107e --- /dev/null +++ b/test/elastix/old/mapping_test.exs @@ -0,0 +1,176 @@ +defmodule Elastix.Old.MappingTest do + use ExUnit.Case + alias Elastix.Index + alias Elastix.Mapping + alias Elastix.Document + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + @test_index2 Elastix.config(:test_index) <> "_2" + @mapping %{ + properties: %{ + user: %{type: "integer"}, + message: %{type: "boolean"} + } + } + @target_mapping %{ + "properties" => %{ + "user" => %{"type" => "integer"}, + "message" => %{"type" => "boolean"} + } + } + @data %{ + user: 12, + message: true + } + @old false + + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = Elastix.HTTP.get(@test_url) + version_string = response.body["version"]["number"] + version = Elastix.version_to_tuple(version_string) + + {:ok, version: version} + end + + setup do + Index.delete(@test_url, @test_index) + Index.delete(@test_url, @test_index2) + + :ok + end + + defp elasticsearch_version do + %HTTPoison.Response{body: %{"version" => %{"number" => v}}, status_code: 200} = + Elastix.HTTP.get!(@test_url) + + v |> String.split([".", "-"]) |> Enum.take(3) |> Enum.map(&String.to_integer/1) + |> List.to_tuple() + end + + test "make_path should make url from index names, types, and query params" do + if @old do + assert Mapping.make_path([@test_index], ["tweet"], version: 34, ttl: "1d") == + "/#{@test_index}/_mapping/tweet?version=34&ttl=1d" + end + end + + test "make_all_path should make url from types, and query params" do + if @old do + assert Mapping.make_all_path(["tweet"], version: 34, ttl: "1d") == + "/_mapping/tweet?version=34&ttl=1d" + end + end + + test "make_all_path should make url from query params" do + if @old do + assert Mapping.make_all_path(version: 34, ttl: "1d") == "/_mapping?version=34&ttl=1d" + end + end + + test "put mapping with no index should error" do + {:ok, response} = Mapping.put(@test_url, @test_index, "message", @mapping) + + assert response.status_code == 404 + end + + test "put should put mapping" do + Index.create(@test_url, @test_index, %{}) + {:ok, response} = Mapping.put(@test_url, @test_index, "message", @mapping) + + assert response.status_code == 200 + assert response.body["acknowledged"] == true + end + + test "get with non existing index should return error" do + {:ok, response} = Mapping.get(@test_url, @test_index, "message") + + assert response.status_code == 404 + end + + test "get with non existing mapping" do + Index.create(@test_url, @test_index, %{}) + {:ok, response} = Mapping.get(@test_url, @test_index, "message") + + if elasticsearch_version() >= {5, 5, 0} do + assert response.body["error"]["reason"] == "type[[message]] missing" + else + assert response.body == %{} + end + + assert response.status_code == 404 + end + + test "get mapping should return mapping" do + Index.create(@test_url, @test_index, %{}) + Mapping.put(@test_url, @test_index, "message", @mapping) + {:ok, response} = Mapping.get(@test_url, @test_index, "message") + + assert response.status_code == 200 + assert response.body[@test_index]["mappings"]["message"] == @target_mapping + end + + test "get mapping for several types should return several mappings", %{version: version} do + Index.create(@test_url, @test_index, %{}) + + if version >= {6, 0, 0} do + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index, "comment", @mapping) + + {:ok, response} = Mapping.get(@test_url, @test_index, ["message", "comment"]) + assert response.status_code == 404 + else + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index, "comment", @mapping) + + {:ok, response} = Mapping.get(@test_url, @test_index, ["message", "comment"]) + assert response.status_code == 200 + assert response.body[@test_index]["mappings"]["message"] == @target_mapping + assert response.body[@test_index]["mappings"]["comment"] == @target_mapping + end + end + + test "get_all mappings should return mappings for all indexes and types" do + Index.create(@test_url, @test_index, %{}) + Index.create(@test_url, @test_index2, %{}) + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index2, "comment", @mapping) + {:ok, response} = Mapping.get_all(@test_url) + + assert response.status_code == 200 + assert response.body[@test_index]["mappings"]["message"] == @target_mapping + assert response.body[@test_index2]["mappings"]["comment"] == @target_mapping + end + + test "get_all_with_type mappings should return mapping for specifieds types in all indexes" do + Index.create(@test_url, @test_index, %{}) + Index.create(@test_url, @test_index2, %{}) + Mapping.put(@test_url, @test_index, "message", @mapping) + Mapping.put(@test_url, @test_index2, "comment", @mapping) + {:ok, response} = Mapping.get_all_with_type(@test_url, ["message", "comment"]) + + assert response.status_code == 200 + assert response.body[@test_index]["mappings"]["message"] == @target_mapping + assert response.body[@test_index2]["mappings"]["comment"] == @target_mapping + end + + test "put document with mapping should put document", %{version: version} do + Index.create(@test_url, @test_index, %{}) + Mapping.put(@test_url, @test_index, "message", @mapping) + + {:ok, response} = Document.index(@test_url, @test_index, "message", 1, @data) + + assert response.status_code == 201 + assert response.body["_id"] == "1" + assert response.body["_index"] == @test_index + assert response.body["_type"] == "message" + + if version >= {6, 0, 0} do + assert response.body["result"] == "created" + else + assert response.body["created"] == true + end + end +end diff --git a/test/elastix/old/search_test.exs b/test/elastix/old/search_test.exs new file mode 100644 index 0000000..459ed28 --- /dev/null +++ b/test/elastix/old/search_test.exs @@ -0,0 +1,117 @@ +defmodule Elastix.Old.SearchTest do + use ExUnit.Case + alias Elastix.Search + alias Elastix.Index + alias Elastix.Document + alias HTTPoison.Response + + @test_url Elastix.config(:test_url) + @test_index Elastix.config(:test_index) + @document_data %{ + user: "werbitzky", + post_date: "2009-11-15T14:12:12", + message: "trying out Elasticsearch" + } + @query_data %{ + query: %{ + term: %{user: "werbitzky"} + } + } + @scroll_query %{ + size: 5, + query: %{match_all: %{}}, + sort: ["_doc"] + } + @old false + + setup do + Index.delete(@test_url, @test_index) + + :ok + end + + test "make_path should make path from id and url" do + if @old do + + path = Search.make_path(@test_index, ["tweet", "product"], ttl: "1d", timeout: 123) + assert path == "/#{@test_index}/tweet,product/_search?ttl=1d&timeout=123" + end + end + + test "make_path should make path that can interchange api type" do + if @old do + path = + Search.make_path( + @test_index, + ["tweet", "product"], + [ttl: "1d", timeout: 123], + "_count" + ) + + assert path == "/#{@test_index}/tweet,product/_count?ttl=1d&timeout=123" + end + end + + test "search should return with status 200" do + Document.index(@test_url, @test_index, "message", 1, @document_data, refresh: true) + + {:ok, response} = Search.search(@test_url, @test_index, [], @query_data) + + assert response.status_code == 200 + end + + test "search accepts httpoison options" do + Document.index(@test_url, @test_index, "message", 1, @document_data, refresh: true) + + {:error, %HTTPoison.Error{reason: :timeout}} = + Search.search(@test_url, @test_index, [], @query_data, [], recv_timeout: 0) + end + + test "search accepts a list of requests" do + Document.index(@test_url, @test_index, "message", 1, @document_data, refresh: true) + Document.index(@test_url, @test_index, "message", 2, @document_data, refresh: true) + + {:ok, response} = + Search.search(@test_url, @test_index, [], [%{}, @query_data, %{}, @query_data]) + + assert [_first, _second] = response.body["responses"] + assert response.status_code == 200 + end + + test "can scroll through all documents" do + for i <- 1..10, + do: + Document.index( + @test_url, + @test_index, + "message", + i, + @document_data, + refresh: true + ) + + {:ok, %Response{body: body}} = + Search.search(@test_url, @test_index, [], @scroll_query, scroll: "1m") + + assert length(body["hits"]["hits"]) === 5 + + {:ok, %Response{body: body}} = + Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) + + assert length(body["hits"]["hits"]) === 5 + + {:ok, %Response{body: body}} = + Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) + + assert length(body["hits"]["hits"]) === 0 + end + + test "count should return with status 200" do + Document.index(@test_url, @test_index, "message", 1, @document_data, refresh: true) + + {:ok, response} = Search.count(@test_url, @test_index, [], @query_data) + + assert response.status_code == 200 + assert response.body["count"] == 1 + end +end diff --git a/test/elastix/old/snapshot/repository_test.exs b/test/elastix/old/snapshot/repository_test.exs new file mode 100644 index 0000000..82a84b3 --- /dev/null +++ b/test/elastix/old/snapshot/repository_test.exs @@ -0,0 +1,112 @@ +defmodule Elastix.Old.Snapshot.RepositoryTest do + @moduledoc """ + Tests for the Elastix.Snapshot.Repository module functions. + + Note that for these tests to run, Elasticsearch must be running and the + elasticsearch.yml file must have the following entry: + + path.repo: /tmp + """ + + use ExUnit.Case + alias Elastix.Snapshot.Repository + + @test_url Elastix.config(:test_url) + @test_repository_config %{"type" => "fs", "settings" => %{"location" => "/tmp/elastix/backups"}} + + @old false + + setup do + on_exit(fn -> + Repository.delete(@test_url, "elastix_test_repository_1") + Repository.delete(@test_url, "elastix_test_repository_2") + end) + + :ok + end + + describe "constructing paths" do + if @old do + test "make_path should make url from repository name and query params" do + assert Repository.make_path("elastix_test_unverified_backup", verify: false) == + "/_snapshot/elastix_test_unverified_backup?verify=false" + end + end + + test "make_path should make url from repository_name" do + assert Repository.make_path("elastix_test_repository_1") == + "/_snapshot/elastix_test_repository_1" + end + end + + describe "registering a repository" do + test "a repository" do + assert {:ok, %{status_code: 200}} = + Repository.register( + @test_url, + "elastix_test_repository_1", + @test_repository_config + ) + end + + test "an unverified repository" do + assert {:ok, %{status_code: 200}} = + Repository.register( + @test_url, + "elastix_test_repository_1", + @test_repository_config, + verify: false + ) + end + end + + describe "verifying a repository" do + test "a registered but unverified repository is manually verified" do + Repository.register( + @test_url, + "elastix_test_repository_1", + @test_repository_config, + verify: false + ) + + assert {:ok, %{status_code: 200, body: %{"nodes" => _}}} = + Repository.verify(@test_url, "elastix_test_repository_1") + end + end + + describe "retrieving information about a repository" do + test "repository doesn't exist" do + assert {:ok, %{status_code: 404}} = Repository.get(@test_url, "nonexistent") + end + + test "information about all repositories" do + Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) + Repository.register(@test_url, "elastix_test_repository_2", @test_repository_config) + + assert {:ok, %{status_code: 200}} = Repository.get(@test_url) + end + + test "information about a specific repository" do + Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) + + assert {:ok, %{status_code: 200}} = + Repository.get(@test_url, "elastix_test_repository_1") + end + end + + describe "deleting a repository" do + test "repository doesn't exist" do + assert {:ok, %{status_code: 404}} = Repository.delete(@test_url, "nonexistent") + end + + test "references to the location where snapshots are stored are removed" do + Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) + + assert {:ok, %{status_code: 200}} = + Repository.delete(@test_url, "elastix_test_repository_1") + + assert {:ok, %{status_code: 404}} = + Repository.get(@test_url, "elastix_test_repository_1") + end + end +end diff --git a/test/elastix/old/snapshot/snapshot_test.exs b/test/elastix/old/snapshot/snapshot_test.exs new file mode 100644 index 0000000..6be2388 --- /dev/null +++ b/test/elastix/old/snapshot/snapshot_test.exs @@ -0,0 +1,304 @@ +defmodule Elastix.Old.Snapshot.SnapshotTest do + @moduledoc """ + Tests for the Elastix.Snapshot.Snapshot module functions. + + Note that for these tests to run, Elasticsearch must be running and the + elasticsearch.yml file must have the following entry: + + path.repo: /tmp + + For testing purposes, snapshots are limited to test indices only. + """ + + use ExUnit.Case + use Retry + alias Elastix.Index + alias Elastix.Snapshot.{Repository, Snapshot} + + @test_url Elastix.config(:test_url) + @test_repository "elastix_test_repository" + @old false + + setup_all do + Index.create(@test_url, "elastix_test_index_1", %{}) + Index.create(@test_url, "elastix_test_index_2", %{}) + Index.create(@test_url, "elastix_test_index_3", %{}) + Index.create(@test_url, "elastix_test_index_4", %{}) + Index.create(@test_url, "elastix_test_index_5", %{}) + + Repository.register(@test_url, @test_repository, %{ + type: "fs", + settings: %{location: "/tmp/elastix/backups"} + }) + + on_exit(fn -> + Index.delete(@test_url, "elastix_test_index_1") + Index.delete(@test_url, "elastix_test_index_2") + Index.delete(@test_url, "elastix_test_index_3") + Index.delete(@test_url, "elastix_test_index_4") + Index.delete(@test_url, "elastix_test_index_5") + + Repository.delete(@test_url, @test_repository) + end) + + :ok + end + + setup do + on_exit(fn -> + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_1") + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_2") + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_3") + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_4") + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_5") + end) + + :ok + end + + describe "constructing paths" do + test "make_path should make url from repository name, snapshot name, and query params" do + if @old do + assert Snapshot.make_path( + @test_repository, + "elastix_test_snapshot_1", + wait_for_completion: true + ) == + "/_snapshot/#{@test_repository}/elastix_test_snapshot_1?wait_for_completion=true" + end + end + + test "make_path should make url from repository name and snapshot name" do + if @old do + assert Snapshot.make_path(@test_repository, "elastix_test_snapshot_1") == + "/_snapshot/#{@test_repository}/elastix_test_snapshot_1" + end + end + end + + describe "creating a snapshot" do + test "a snapshot of multiple indices in the cluster" do + Snapshot.create( + @test_url, + @test_repository, + "elastix_test_snapshot_2", + %{indices: "elastix_test_index_1,elastix_test_index_2"}, + wait_for_completion: true + ) + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{body: %{"snapshots" => snapshots}}} = + Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_2") + + snapshot = List.first(snapshots) + snapshot["state"] == "SUCCESS" + ) + + then + + ( + {:ok, %{body: %{"snapshots" => snapshots}}} = + Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_2") + + snapshot = List.first(snapshots) + assert Enum.member?(snapshot["indices"], "elastix_test_index_1") + assert Enum.member?(snapshot["indices"], "elastix_test_index_2") + ) + end + end + + test "a snapshot of a single index in the cluster" do + Snapshot.create( + @test_url, + @test_repository, + "elastix_test_snapshot_1", + %{indices: "elastix_test_index_1"}, + wait_for_completion: true + ) + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{body: %{"snapshots" => snapshots}}} = + Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_1") + + snapshot = List.first(snapshots) + snapshot["state"] == "SUCCESS" + ) + + then + + ( + {:ok, %{body: %{"snapshots" => snapshots}}} = + Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_1") + + snapshot = List.first(snapshots) + assert Enum.member?(snapshot["indices"], "elastix_test_index_1") + refute Enum.member?(snapshot["indices"], "elastix_test_index_2") + ) + end + end + end + + describe "restoring a snapshot" do + test "all indices in a snapshot" do + Snapshot.create( + @test_url, + @test_repository, + "elastix_test_snapshot_4", + %{indices: "elastix_test_index_1,elastix_test_index_2"}, + wait_for_completion: true + ) + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{body: %{"snapshots" => snapshots}}} = + Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_4") + + snapshot = List.first(snapshots) + snapshot["state"] == "SUCCESS" + ) + + then + + ( + Index.close(@test_url, "elastix_test_index_1") + Index.close(@test_url, "elastix_test_index_2") + Index.delete(@test_url, "elastix_test_index_1") + Index.delete(@test_url, "elastix_test_index_2") + ) + end + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_1") + {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_2") + ) + + then + + Snapshot.restore(@test_url, @test_repository, "elastix_test_snapshot_4", %{ + partial: true + }) + end + + wait lin_backoff(500, 1) |> expiry(5_000) do + {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_1") + {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_2") + end + end + + test "a specific index in a snapshot" do + Snapshot.create( + @test_url, + @test_repository, + "elastix_test_snapshot_3", + %{indices: "elastix_test_index_3,elastix_test_index_4"}, + wait_for_completion: true + ) + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{status_code: 200, body: %{"snapshots" => snapshots}}} = + Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_3") + + snapshot = List.first(snapshots) + snapshot["state"] == "SUCCESS" + ) + + then + + ( + Index.close(@test_url, "elastix_test_index_3") + Index.close(@test_url, "elastix_test_index_4") + Index.delete(@test_url, "elastix_test_index_3") + Index.delete(@test_url, "elastix_test_index_4") + ) + end + + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_3") + {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_4") + ) + + then + + Snapshot.restore(@test_url, @test_repository, "elastix_test_snapshot_3", %{ + indices: "elastix_test_index_3" + }) + end + + wait lin_backoff(500, 1) |> expiry(5_000) do + {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_3") + {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_4") + end + end + end + + describe "retrieving status information for a snapshot" do + test "snapshot doesn't exist" do + assert {:ok, %{status_code: 404}} = + Snapshot.status(@test_url, @test_repository, "nonexistent") + end + + test "information about all snapshots" do + assert {:ok, %{status_code: 200}} = Snapshot.status(@test_url) + end + + test "information about all snapshots in a repository" do + assert {:ok, %{status_code: 200}} = Snapshot.status(@test_url, @test_repository) + end + + test "information about a specific snapshot" do + Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ + indices: "elastix_test_index_5" + }) + + wait lin_backoff(500, 1) |> expiry(5_000) do + {:ok, %{status_code: 200}} = + Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_5") + end + end + end + + describe "retrieving information about a snapshot" do + test "snapshot doesn't exist" do + assert {:ok, %{status_code: 404}} = + Snapshot.get(@test_url, @test_repository, "nonexistent") + end + + test "information about all snapshots" do + assert {:ok, %{status_code: 200}} = Snapshot.get(@test_url, @test_repository) + end + + test "information about a specific snapshot" do + Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ + indices: "elastix_test_index_5" + }) + + assert {:ok, %{status_code: 200}} = + Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_5") + end + end + + describe "deleting a snapshot" do + test "snapshot doesn't exist" do + assert {:ok, %{status_code: 404}} = + Snapshot.delete(@test_url, @test_repository, "nonexistent") + end + + test "snapshot is deleted" do + Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ + indices: "elastix_test_index_5" + }) + + # This needs a wait to make sure that the restore completes before deleting + assert {:ok, %{status_code: 200}} = + Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_5") + + assert {:ok, %{status_code: 404}} = + Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_5") + end + end +end diff --git a/test/elastix/search_test.exs b/test/elastix/search_test.exs index 84bb2e4..55c42ed 100644 --- a/test/elastix/search_test.exs +++ b/test/elastix/search_test.exs @@ -3,7 +3,6 @@ defmodule Elastix.SearchTest do alias Elastix.Search alias Elastix.Index alias Elastix.Document - alias HTTPoison.Response @test_url Elastix.config(:test_url) @test_index Elastix.config(:test_index) @@ -31,21 +30,16 @@ defmodule Elastix.SearchTest do :ok end - test "make_path should make path from id and url" do - path = Search.make_path(@test_index, ["tweet", "product"], ttl: "1d", timeout: 123) - assert path == "/#{@test_index}/tweet,product/_search?ttl=1d&timeout=123" - end - - test "make_path should make path that can interchange api type" do - path = - Search.make_path( - @test_index, - ["tweet", "product"], - [ttl: "1d", timeout: 123], - "_count" - ) + describe "make_path/2" do + test "makes path from index and types" do + path = Search.make_path(@test_index, ["tweet", "product"]) + assert path == "/#{@test_index}/tweet,product/_search" + end - assert path == "/#{@test_index}/tweet,product/_count?ttl=1d&timeout=123" + test "makes path with API type" do + path = Search.make_path(@test_index, ["tweet", "product"], "_count") + assert path == "/#{@test_index}/tweet,product/_count" + end end test "search should return with status 200" do @@ -85,35 +79,30 @@ defmodule Elastix.SearchTest do assert response.status_code == 200 end - test "can scroll through all documents" do - for i <- 1..10, - do: - Document.index( - @test_url, - @test_index, - "message", - i, - @document_data, - refresh: true - ) + describe "scroll/3" do - {:ok, %Response{body: body}} = - Search.search(@test_url, @test_index, [], @scroll_query, scroll: "1m") + test "can scroll through results" do + for i <- 1..10 do + Document.index(@test_url, @test_index, @document_data, %{id: i}, refresh: true) + end - assert length(body["hits"]["hits"]) === 5 + {:ok, %{status_code: 200, body: body}} = + Search.search(@test_url, @test_index, [], @scroll_query, scroll: "1m") + assert length(body["hits"]["hits"]) === 5 - {:ok, %Response{body: body}} = - Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) + {:ok, %{status_code: 200, body: body}} = + Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) + assert length(body["hits"]["hits"]) === 5 - assert length(body["hits"]["hits"]) === 5 + {:ok, %{status_code: 200, body: body}} = + Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) - {:ok, %Response{body: body}} = - Search.scroll(@test_url, %{scroll: "1m", scroll_id: body["_scroll_id"]}) + assert length(body["hits"]["hits"]) === 0 + end - assert length(body["hits"]["hits"]) === 0 end - test "count should return with status 200" do + test "count returns status 200" do Document.index(@test_url, @test_index, "message", 1, @document_data, refresh: true) {:ok, response} = Search.count(@test_url, @test_index, [], @query_data) diff --git a/test/elastix/snapshot/repository_test.exs b/test/elastix/snapshot/repository_test.exs index 7c2f40c..ccec6cd 100644 --- a/test/elastix/snapshot/repository_test.exs +++ b/test/elastix/snapshot/repository_test.exs @@ -3,90 +3,97 @@ defmodule Elastix.Snapshot.RepositoryTest do Tests for the Elastix.Snapshot.Repository module functions. Note that for these tests to run, Elasticsearch must be running and the - elasticsearch.yml file must have the following entry: + config file `elasticsearch.yml` file must have the following entry: - path.repo: /tmp + `path.repo: ["/tmp/elastix/backups"]` + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#_shared_file_system_repository) """ use ExUnit.Case alias Elastix.Snapshot.Repository + alias Elastix.HTTP @test_url Elastix.config(:test_url) - @test_repository_config %{type: "fs", settings: %{location: "/tmp"}} + @repo_config %{"type" => "fs", "settings" => %{"location" => "/tmp/elastix/backups"}} + @repo_1 "elastix_test_repository_1" + @repo_2 "elastix_test_repository_2" + + setup_all do + # Query the Elasticsearch instance to determine what version it is running + # so we can use it in tests. + {:ok, response} = HTTP.get(@test_url) + version_string = response.body["version"]["number"] + version = Elastix.version_to_tuple(version_string) + + {:ok, version: version} + end setup do on_exit(fn -> - Repository.delete(@test_url, "elastix_test_repository_1") - Repository.delete(@test_url, "elastix_test_repository_2") + Repository.delete(@test_url, @repo_1) + Repository.delete(@test_url, @repo_2) end) :ok end - describe "constructing paths" do - test "make_path should make url from repository name and query params" do - assert Repository.make_path("elastix_test_unverified_backup", verify: false) == - "/_snapshot/elastix_test_unverified_backup?verify=false" - end - - test "make_path should make url from repository_name" do - assert Repository.make_path("elastix_test_repository_1") == - "/_snapshot/elastix_test_repository_1" + describe "make_path/2" do + test "handles all parameter variations" do + assert Repository.make_path(nil) == "/_snapshot" + assert Repository.make_path("foo") == "/_snapshot/foo" end end describe "registering a repository" do test "a repository" do - assert {:ok, %{status_code: 200}} = - Repository.register( - @test_url, - "elastix_test_repository_1", - @test_repository_config - ) - end - - test "an unverified repository" do - assert {:ok, %{status_code: 200}} = - Repository.register( - @test_url, - "elastix_test_repository_1", - @test_repository_config, - verify: false - ) + {:ok, response} = Repository.register(@test_url, @repo_1, @repo_config) + assert response.status_code == 200 end end describe "verifying a repository" do test "a registered but unverified repository is manually verified" do - Repository.register( - @test_url, - "elastix_test_repository_1", - @test_repository_config, - verify: false - ) - - assert {:ok, %{status_code: 200, body: %{"nodes" => _}}} = - Repository.verify(@test_url, "elastix_test_repository_1") + {:ok, response} = Repository.register(@test_url, @repo_2, @repo_config, verify: false) + assert response.status_code == 200 + + {:ok, response} = Repository.verify(@test_url, @repo_2) + assert response.status_code == 200 + assert response.body["nodes"] != "" end end describe "retrieving information about a repository" do test "repository doesn't exist" do - assert {:ok, %{status_code: 404}} = Repository.get(@test_url, "nonexistent") + {:ok, response} = Repository.get(@test_url, "nonexistent") + assert response.status_code == 404 end test "information about all repositories" do - Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) - Repository.register(@test_url, "elastix_test_repository_2", @test_repository_config) + {:ok, %{status_code: 200}} = Repository.register(@test_url, @repo_1, @repo_config) + {:ok, %{status_code: 200}} = Repository.register(@test_url, @repo_2, @repo_config) - assert {:ok, %{status_code: 200}} = Repository.get(@test_url) + {:ok, response} = Repository.get(@test_url) + assert response.status_code == 200 + assert response.body[@repo_1] == @repo_config + assert response.body[@repo_2] == @repo_config end test "information about a specific repository" do - Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) + Repository.register(@test_url, @repo_1, @repo_config) + {:ok, response} = Repository.get(@test_url, @repo_1) + assert response.status_code == 200 + assert response.body[@repo_1] == @repo_config + end + end - assert {:ok, %{status_code: 200}} = - Repository.get(@test_url, "elastix_test_repository_1") + describe "cleanup" do + test "repository", %{version: version} do + if version >= {7, 0, 0} do + {:ok, %{status_code: 200}} = Repository.register(@test_url, @repo_1, @repo_config) + {:ok, response} = Repository.cleanup(@test_url, @repo_1) + assert response.status_code == 200 + end end end @@ -96,13 +103,10 @@ defmodule Elastix.Snapshot.RepositoryTest do end test "references to the location where snapshots are stored are removed" do - Repository.register(@test_url, "elastix_test_repository_1", @test_repository_config) - - assert {:ok, %{status_code: 200}} = - Repository.delete(@test_url, "elastix_test_repository_1") + assert {:ok, %{status_code: 200}} = Repository.register(@test_url, @repo_1, @repo_config) - assert {:ok, %{status_code: 404}} = - Repository.get(@test_url, "elastix_test_repository_1") + assert {:ok, %{status_code: 200}} = Repository.delete(@test_url, @repo_1) + assert {:ok, %{status_code: 404}} = Repository.get(@test_url, @repo_1) end end end diff --git a/test/elastix/snapshot/snapshot_test.exs b/test/elastix/snapshot/snapshot_test.exs index 8dd8709..941d9b5 100644 --- a/test/elastix/snapshot/snapshot_test.exs +++ b/test/elastix/snapshot/snapshot_test.exs @@ -3,9 +3,11 @@ defmodule Elastix.Snapshot.SnapshotTest do Tests for the Elastix.Snapshot.Snapshot module functions. Note that for these tests to run, Elasticsearch must be running and the - elasticsearch.yml file must have the following entry: + config file `elasticsearch.yml` file must have the following entry: - path.repo: /tmp + `path.repo: ["/tmp/elastix/backups"]` + + [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#_shared_file_system_repository) For testing purposes, snapshots are limited to test indices only. """ @@ -16,28 +18,37 @@ defmodule Elastix.Snapshot.SnapshotTest do alias Elastix.Snapshot.{Repository, Snapshot} @test_url Elastix.config(:test_url) - @test_repository "elastix_test_repository" + @repo "elastix_test_repository" + @repo_config %{type: "fs", settings: %{location: "/tmp/elastix/backups"}} + @index_1 "elastix_test_index_1" + @index_2 "elastix_test_index_2" + @index_3 "elastix_test_index_3" + @index_4 "elastix_test_index_4" + @index_5 "elastix_test_index_5" + + @snapshot_1 "elastix_test_snapshot_1" + @snapshot_2 "elastix_test_snapshot_2" + @snapshot_3 "elastix_test_snapshot_3" + @snapshot_4 "elastix_test_snapshot_4" + @snapshot_5 "elastix_test_snapshot_5" setup_all do - Index.create(@test_url, "elastix_test_index_1", %{}) - Index.create(@test_url, "elastix_test_index_2", %{}) - Index.create(@test_url, "elastix_test_index_3", %{}) - Index.create(@test_url, "elastix_test_index_4", %{}) - Index.create(@test_url, "elastix_test_index_5", %{}) + Index.create(@test_url, @index_1, %{}) + Index.create(@test_url, @index_2, %{}) + Index.create(@test_url, @index_3, %{}) + Index.create(@test_url, @index_4, %{}) + Index.create(@test_url, @index_5, %{}) - Repository.register(@test_url, @test_repository, %{ - type: "fs", - settings: %{location: "/tmp"} - }) + Repository.register(@test_url, @repo, @repo_config) on_exit(fn -> - Index.delete(@test_url, "elastix_test_index_1") - Index.delete(@test_url, "elastix_test_index_2") - Index.delete(@test_url, "elastix_test_index_3") - Index.delete(@test_url, "elastix_test_index_4") - Index.delete(@test_url, "elastix_test_index_5") + Index.delete(@test_url, @index_1) + Index.delete(@test_url, @index_2) + Index.delete(@test_url, @index_3) + Index.delete(@test_url, @index_4) + Index.delete(@test_url, @index_5) - Repository.delete(@test_url, @test_repository) + Repository.delete(@test_url, @repo) end) :ok @@ -45,91 +56,65 @@ defmodule Elastix.Snapshot.SnapshotTest do setup do on_exit(fn -> - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_1") - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_2") - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_3") - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_4") - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_5") + Snapshot.delete(@test_url, @repo, @snapshot_1) + Snapshot.delete(@test_url, @repo, @snapshot_2) + Snapshot.delete(@test_url, @repo, @snapshot_3) + Snapshot.delete(@test_url, @repo, @snapshot_4) + Snapshot.delete(@test_url, @repo, @snapshot_5) end) :ok end describe "constructing paths" do - test "make_path should make url from repository name, snapshot name, and query params" do - assert Snapshot.make_path( - @test_repository, - "elastix_test_snapshot_1", - wait_for_completion: true - ) == - "/_snapshot/#{@test_repository}/elastix_test_snapshot_1?wait_for_completion=true" - end - - test "make_path should make url from repository name and snapshot name" do - assert Snapshot.make_path(@test_repository, "elastix_test_snapshot_1") == - "/_snapshot/#{@test_repository}/elastix_test_snapshot_1" + test "make_path/2 makes path from repository name and snapshot name" do + assert Snapshot.make_path(@repo, @snapshot_1) == "/_snapshot/#{@repo}/#{@snapshot_1}" end end describe "creating a snapshot" do test "a snapshot of multiple indices in the cluster" do - Snapshot.create( - @test_url, - @test_repository, - "elastix_test_snapshot_2", - %{indices: "elastix_test_index_1,elastix_test_index_2"}, - wait_for_completion: true - ) + {:ok, response} = Snapshot.create(@test_url, @repo, @snapshot_2, + %{indices: "#{@index_1},#{@index_2}"}, wait_for_completion: true) + assert response.status_code == 200 wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{body: %{"snapshots" => snapshots}}} = - Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_2") - - snapshot = List.first(snapshots) + {:ok, response} = Snapshot.status(@test_url, @repo, @snapshot_2) + snapshot = List.first(response.body["snapshots"]) snapshot["state"] == "SUCCESS" ) then ( - {:ok, %{body: %{"snapshots" => snapshots}}} = - Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_2") - - snapshot = List.first(snapshots) - assert Enum.member?(snapshot["indices"], "elastix_test_index_1") - assert Enum.member?(snapshot["indices"], "elastix_test_index_2") + {:ok, response} = Snapshot.get(@test_url, @repo, @snapshot_2) + snapshot = List.first(response.body["snapshots"]) + assert Enum.member?(snapshot["indices"], @index_1) + assert Enum.member?(snapshot["indices"], @index_2) ) end end test "a snapshot of a single index in the cluster" do - Snapshot.create( - @test_url, - @test_repository, - "elastix_test_snapshot_1", - %{indices: "elastix_test_index_1"}, - wait_for_completion: true - ) + {:ok, response} = Snapshot.create(@test_url, @repo, @snapshot_1, + %{indices: @index_1}, wait_for_completion: true) + assert response.status_code == 200 wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{body: %{"snapshots" => snapshots}}} = - Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_1") - - snapshot = List.first(snapshots) + {:ok, response} = Snapshot.status(@test_url, @repo, @snapshot_1) + snapshot = List.first(response.body["snapshots"]) snapshot["state"] == "SUCCESS" ) then ( - {:ok, %{body: %{"snapshots" => snapshots}}} = - Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_1") - - snapshot = List.first(snapshots) - assert Enum.member?(snapshot["indices"], "elastix_test_index_1") - refute Enum.member?(snapshot["indices"], "elastix_test_index_2") + {:ok, response} = Snapshot.get(@test_url, @repo, @snapshot_1) + snapshot = List.first(response.body["snapshots"]) + assert Enum.member?(snapshot["indices"], @index_1) + refute Enum.member?(snapshot["indices"], @index_2) ) end end @@ -137,65 +122,54 @@ defmodule Elastix.Snapshot.SnapshotTest do describe "restoring a snapshot" do test "all indices in a snapshot" do - Snapshot.create( - @test_url, - @test_repository, - "elastix_test_snapshot_4", - %{indices: "elastix_test_index_1,elastix_test_index_2"}, - wait_for_completion: true - ) + {:ok, response} = Snapshot.create(@test_url, @repo, @snapshot_4, + %{indices: "#{@index_1},#{@index_2}"}, wait_for_completion: true) + assert response.status_code == 200 wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{body: %{"snapshots" => snapshots}}} = - Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_4") - - snapshot = List.first(snapshots) + {:ok, response} = Snapshot.status(@test_url, @repo, @snapshot_4) + snapshot = List.first(response.body["snapshots"]) snapshot["state"] == "SUCCESS" ) then ( - Index.close(@test_url, "elastix_test_index_1") - Index.close(@test_url, "elastix_test_index_2") - Index.delete(@test_url, "elastix_test_index_1") - Index.delete(@test_url, "elastix_test_index_2") + Index.close(@test_url, @index_1) + Index.close(@test_url, @index_2) + Index.delete(@test_url, @index_1) + Index.delete(@test_url, @index_2) ) end wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_1") - {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_2") + {:ok, %{status_code: 404}} = Index.get(@test_url, @index_1) + {:ok, %{status_code: 404}} = Index.get(@test_url, @index_2) ) then - Snapshot.restore(@test_url, @test_repository, "elastix_test_snapshot_4", %{ - partial: true - }) + Snapshot.restore(@test_url, @repo, @snapshot_4, %{partial: true}) end wait lin_backoff(500, 1) |> expiry(5_000) do - {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_1") - {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_2") + {:ok, %{status_code: 200}} = Index.get(@test_url, @index_1) + {:ok, %{status_code: 200}} = Index.get(@test_url, @index_2) end end test "a specific index in a snapshot" do - Snapshot.create( - @test_url, - @test_repository, - "elastix_test_snapshot_3", - %{indices: "elastix_test_index_3,elastix_test_index_4"}, - wait_for_completion: true - ) + {:ok, response} = Snapshot.create(@test_url, @repo, @snapshot_3, + %{indices: Enum.join([@index_3, @index_4], ",")}, wait_for_completion: true) + assert response.status_code == 200 wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{status_code: 200, body: %{"snapshots" => snapshots}}} = - Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_3") + {:ok, response} = Snapshot.status(@test_url, @repo, @snapshot_3) + assert response.status_code == 200 + snapshots = response.body["snapshots"] snapshot = List.first(snapshots) snapshot["state"] == "SUCCESS" @@ -204,37 +178,34 @@ defmodule Elastix.Snapshot.SnapshotTest do then ( - Index.close(@test_url, "elastix_test_index_3") - Index.close(@test_url, "elastix_test_index_4") - Index.delete(@test_url, "elastix_test_index_3") - Index.delete(@test_url, "elastix_test_index_4") + Index.close(@test_url, @index_3) + Index.close(@test_url, @index_4) + Index.delete(@test_url, @index_3) + Index.delete(@test_url, @index_4) ) end wait lin_backoff(500, 1) |> expiry(5_000) do ( - {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_3") - {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_4") + {:ok, %{status_code: 404}} = Index.get(@test_url, @index_3) + {:ok, %{status_code: 404}} = Index.get(@test_url, @index_4) ) then - Snapshot.restore(@test_url, @test_repository, "elastix_test_snapshot_3", %{ - indices: "elastix_test_index_3" - }) + Snapshot.restore(@test_url, @repo, @snapshot_3, %{indices: @index_3}) end wait lin_backoff(500, 1) |> expiry(5_000) do - {:ok, %{status_code: 200}} = Index.get(@test_url, "elastix_test_index_3") - {:ok, %{status_code: 404}} = Index.get(@test_url, "elastix_test_index_4") + {:ok, %{status_code: 200}} = Index.get(@test_url, @index_3) + {:ok, %{status_code: 404}} = Index.get(@test_url, @index_4) end end end describe "retrieving status information for a snapshot" do test "snapshot doesn't exist" do - assert {:ok, %{status_code: 404}} = - Snapshot.status(@test_url, @test_repository, "nonexistent") + assert {:ok, %{status_code: 404}} = Snapshot.status(@test_url, @repo, "nonexistent") end test "information about all snapshots" do @@ -242,57 +213,55 @@ defmodule Elastix.Snapshot.SnapshotTest do end test "information about all snapshots in a repository" do - assert {:ok, %{status_code: 200}} = Snapshot.status(@test_url, @test_repository) + assert {:ok, %{status_code: 200}} = Snapshot.status(@test_url, @repo) end test "information about a specific snapshot" do - Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ - indices: "elastix_test_index_5" - }) + Snapshot.create(@test_url, @repo, @snapshot_5, %{indices: @index_5}) wait lin_backoff(500, 1) |> expiry(5_000) do - {:ok, %{status_code: 200}} = - Snapshot.status(@test_url, @test_repository, "elastix_test_snapshot_5") + {:ok, %{status_code: 200}} = Snapshot.status(@test_url, @repo, @snapshot_5) end end end describe "retrieving information about a snapshot" do test "snapshot doesn't exist" do - assert {:ok, %{status_code: 404}} = - Snapshot.get(@test_url, @test_repository, "nonexistent") + assert {:ok, %{status_code: 404}} = Snapshot.get(@test_url, @repo, "nonexistent") end test "information about all snapshots" do - assert {:ok, %{status_code: 200}} = Snapshot.get(@test_url, @test_repository) + assert {:ok, %{status_code: 200}} = Snapshot.get(@test_url, @repo) end test "information about a specific snapshot" do - Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ - indices: "elastix_test_index_5" - }) - - assert {:ok, %{status_code: 200}} = - Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_5") + Snapshot.create(@test_url, @repo, @snapshot_5, %{indices: @index_5}) + assert {:ok, %{status_code: 200}} = Snapshot.get(@test_url, @repo, @snapshot_5) end end describe "deleting a snapshot" do test "snapshot doesn't exist" do - assert {:ok, %{status_code: 404}} = - Snapshot.delete(@test_url, @test_repository, "nonexistent") + assert {:ok, %{status_code: 404}} = Snapshot.delete(@test_url, @repo, "nonexistent") end test "snapshot is deleted" do - Snapshot.create(@test_url, @test_repository, "elastix_test_snapshot_5", %{ - indices: "elastix_test_index_5" - }) + Snapshot.create(@test_url, @repo, @snapshot_5, %{indices: @index_5}) - assert {:ok, %{status_code: 200}} = - Snapshot.delete(@test_url, @test_repository, "elastix_test_snapshot_5") + wait lin_backoff(500, 1) |> expiry(5_000) do + ( + {:ok, response} = Snapshot.status(@test_url, @repo, @snapshot_5) + snapshot = List.first(response.body["snapshots"]) + snapshot["state"] == "SUCCESS" + ) - assert {:ok, %{status_code: 404}} = - Snapshot.get(@test_url, @test_repository, "elastix_test_snapshot_5") + then + + ( + assert {:ok, %{status_code: 200}} = Snapshot.delete(@test_url, @repo, @snapshot_5) + assert {:ok, %{status_code: 404}} = Snapshot.get(@test_url, @repo, @snapshot_5) + ) + end end end end diff --git a/test/elastix/template_test.exs b/test/elastix/template_test.exs new file mode 100644 index 0000000..ae57d8a --- /dev/null +++ b/test/elastix/template_test.exs @@ -0,0 +1,96 @@ +defmodule Elastix.TemplateTest do + + use ExUnit.Case + + alias Elastix.Template + + # @test_url Elastix.config(:test_url) + + @test_url Elastix.config(:test_url) + @template "elastix_test" + + setup do + on_exit(fn -> + Template.delete(@test_url, @template) + end) + + template = """ + { + "template" : "logstash-*", + "version" : 60001, + "settings" : { + "index.refresh_interval" : "5s" + }, + "mappings" : { + "_default_" : { + "dynamic_templates" : [ { + "message_field" : { + "path_match" : "message", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", + "norms" : false + } + } + }, { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", "norms" : false, + "fields" : { + "keyword" : { "type": "keyword", "ignore_above": 256 } + } + } + } + } ], + "properties" : { + "@timestamp": { "type": "date"}, + "@version": { "type": "keyword"}, + "geoip" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } + } + } + } + } + } + """ + + {:ok, template_data: template} + end + + test "make_path/1 makes path from params" do + assert Template.make_path("foo") == "/_template/foo" + end + + test "creates template", %{template_data: template_data} do + {:ok, response} = Template.put(@test_url, @template, template_data) + assert response.status_code == 200 + end + + test "gets template info", %{template_data: template_data} do + {:ok, response} = Template.put(@test_url, @template, template_data) + assert response.status_code == 200 + + {:ok, response} = Template.get(@test_url, @template) + assert response.status_code == 200 + assert response.body[@template]["index_patterns"] == ["logstash-*"] + end + + test "checks if template exists", %{template_data: template_data} do + assert {:ok, false} == Template.exists?(@test_url, @template) + + {:ok, response} = Template.put(@test_url, @template, template_data) + assert response.status_code == 200 + + assert {:ok, true} == Template.exists?(@test_url, @template) + end + +end + diff --git a/test/elastix_test.exs b/test/elastix_test.exs index ea6602a..1690ca7 100644 --- a/test/elastix_test.exs +++ b/test/elastix_test.exs @@ -1,3 +1,10 @@ defmodule ElastixTest do use ExUnit.Case + + test "version_to_tuple/1" do + assert Elastix.version_to_tuple("1.0") == {1, 0, 0} + assert Elastix.version_to_tuple("1.1") == {1, 1, 0} + assert Elastix.version_to_tuple("1.2.3") == {1, 2, 3} + end + end