Add support for Elasticsearch 7.0 and templates

README.md

@@ -7,7 +7,16 @@
 [![WTFPL](https://img.shields.io/badge/license-WTFPL-brightgreen.svg?style=flat)](https://www.tldrlegal.com/l/wtfpl)
 [![Last Updated](https://img.shields.io/github/last-commit/werbitzky/elastix.svg)](https://github.com/werbitzky/elastix/commits/master)
 
-A DSL-free Elasticsearch client for Elixir.
+An Elasticsearch client for Elixir.
+
+This library has convenience functions for working with the Elasticsearch
+API. It does not use a DSL, it expects you to interact with the Elasticsearch
+using native JSON data structures, or the equivalent Elixir data structures which
+it will encode for you.
+
+It supports the new Elasticsearch 7.x as well as older versions.
+7.x has [removed mapping types on indexes](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html),
+which resulted in incompatible changes to a number of APIs.
 
 ## Documentation
 
@@ -34,7 +43,7 @@ Then run `mix deps.get` to fetch the new dependency.
 
 ## Examples
 
-### Creating an Elasticsearch index
+### Creating an index
 
 ```elixir
 Elastix.Index.create("http://localhost:9200", "twitter", %{})
@@ -46,9 +55,9 @@ Elastix.Index.create("http://localhost:9200", "twitter", %{})
 elastic_url = "http://localhost:9200"
 
 data = %{
-    user: "kimchy",
-    post_date: "2009-11-15T14:12:12",
-    message: "trying out Elastix"
+  user: "kimchy",
+  post_date: "2009-11-15T14:12:12",
+  message: "trying out Elastix"
 }
 
 mapping = %{
@@ -67,7 +76,9 @@ Elastix.Document.delete(elastic_url, "twitter", "tweet", "42")
 
 ### Bulk requests
 
-Bulk requests take as parameter a list of the lines you want to send to the [`_bulk`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) endpoint.
+Bulk requests take as parameter a list of the lines you want to send to the
+[`_bulk`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html)
+endpoint.
 
 You can also specify the following options:
 
@@ -109,7 +120,9 @@ config :elastix,
   httpoison_options: [hackney: [pool: :elastix_pool]]
 ```
 
-Note that you can configure Elastix to use any JSON library, see the ["Custom JSON codec" page](https://hexdocs.pm/elastix/custom-json-codec.html) for more info.
+Note that you can configure Elastix to use any JSON library, see the
+["Custom JSON codec" page](https://hexdocs.pm/elastix/custom-json-codec.html) for more
+info.
 
 ### Custom headers
 
@@ -118,7 +131,10 @@ config :elastix,
   custom_headers: {MyModule, :add_aws_signature, ["us-east"]}
 ```
 
-`custom_headers` must be a tuple of the type `{Module, :function, [args]}`, where `:function` is a function that should accept the request (a map of this type: `%{method: String.t, headers: [], url: String.t, body: String.t}`) as its first parameter and return a list of the headers you want to send:
+`custom_headers` must be a tuple of the type `{Module, :function, [args]}`,
+where `:function` is a function that should accept the request (a map of this
+type: `%{method: String.t, headers: [], url: String.t, body: String.t}`) as its
+first parameter and return a list of the headers you want to send:
 
 ```elixir
 defmodule MyModule do
@@ -134,7 +150,8 @@ end
 
 ## Running tests
 
-You need Elasticsearch running locally on port 9200. A quick way of doing so is via Docker:
+You need Elasticsearch running locally on port 9200.
+A quick way of doing so is via Docker:
 
 ```
 $ docker run -p 9200:9200 -it --rm elasticsearch:5.1.2
@@ -151,6 +168,8 @@ $ mix test
 
 ## Copyright and License
 
-Copyright © 2017 El Werbitzky werbitzky@gmail.com
+Copyright © 2017-2019 El Werbitzky werbitzky@gmail.com
 
-This work is free. You can redistribute it and/or modify it under the terms of the Do What The Fuck You Want To Public License, Version 2, as published by Sam Hocevar. See http://www.wtfpl.net/ for more details.
+This work is free. You can redistribute it and/or modify it under the terms of
+the Do What The Fuck You Want To Public License, Version 2, as published by Sam
+Hocevar. See http://www.wtfpl.net/ for more details.

lib/elastix.ex

@@ -19,4 +19,16 @@ defmodule Elastix do
 
   @doc false
   def config(key, default \\ nil), do: Application.get_env(:elastix, key, default)
+
+
+  @doc "Convert Elasticsearch version string into tuple."
+  def version_to_tuple(version) when is_binary(version) do
+    version
+    |> String.split([".", "-"])
+    |> Enum.map(&String.to_integer/1)
+    |> version_to_tuple()
+  end
+  def version_to_tuple([first, second]), do: {first, second, 0}
+  def version_to_tuple([first, second, third | _rest]), do: {first, second, third}
+
 end

lib/elastix/alias.ex

@@ -1,24 +1,28 @@
 defmodule Elastix.Alias do
   @moduledoc """
-  The alias API makes it possible to perform alias operations on indexes.
+  The alias API adds or removes index aliases, a secondary name used to refer
+  to one or more existing indices.
 
-  [Aliases documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html)
+  [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html)
   """
-  import Elastix.HTTP, only: [prepare_url: 2]
   alias Elastix.{HTTP, JSON}
 
   @doc """
-  Excepts a list of actions for the `actions` parameter.
+  Perform actions on aliases.
+
+  Specify the list of actions in the `actions` parameter.
 
   ## Examples
-      iex> actions = [%{ add: %{ index: "test1", alias: "alias1" }}]
+
+      iex> actions = [%{add: %{index: "test1", alias: "alias1"}}]
       iex> Elastix.Alias.post("http://localhost:9200", actions)
       {:ok, %HTTPoison.Response{...}}
+
   """
-  @spec post(elastic_url :: String.t(), actions :: list) :: HTTP.resp()
+  @spec post(binary, [map]) :: HTTP.resp
   def post(elastic_url, actions) do
-    prepare_url(elastic_url, ["_aliases"])
-    |> HTTP.post(JSON.encode!(%{actions: actions}))
+    url = HTTP.make_url(elastic_url, "_aliases")
+    HTTP.post(url, JSON.encode!(%{actions: actions}))
   end
 
   @doc """

lib/elastix/bulk.ex

@@ -1,94 +1,69 @@
 defmodule Elastix.Bulk do
   @moduledoc """
-  The bulk API makes it possible to perform many index/delete operations in a single API call.
+  The bulk API performs multiple indexing or delete operations in a single API
+  call. This reduces overhead and can greatly increase indexing speed.
 
   [Elastic documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html)
   """
-  import Elastix.HTTP, only: [prepare_url: 2]
   alias Elastix.{HTTP, JSON}
 
+  require Logger
+
   @doc """
-  Excepts a list of actions and sources for the `lines` parameter.
+  Send a batch of actions and sources, encoding them as JSON.
+
+  Data should be encoded as a list of maps.
+  See the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html).
 
   ## Examples
 
-      iex> Elastix.Bulk.post("http://localhost:9200", [%{index: %{_id: "1"}}, %{user: "kimchy"}], index: "twitter", type: "tweet")
+      iex> operations = [%{index: %{_id: "1"}}, %{user: "kimchy"}]
+      iex> Elastix.Bulk.post("http://localhost:9200", operations, index: "twitter", type: "tweet")
       {:ok, %HTTPoison.Response{...}}
-  """
-  @spec post(
-          elastic_url :: String.t(),
-          lines :: list,
-          opts :: Keyword.t(),
-          query_params :: Keyword.t()
-        ) :: HTTP.resp()
-  def post(elastic_url, lines, options \\ [], query_params \\ []) do
-    data =
-      Enum.reduce(lines, [], fn l, acc -> ["\n", JSON.encode!(l) | acc] end)
-      |> Enum.reverse()
-      |> IO.iodata_to_binary()
-
-    path =
-      Keyword.get(options, :index)
-      |> make_path(Keyword.get(options, :type), query_params)
-
-    httpoison_options = Keyword.get(options, :httpoison_options, [])
 
-    elastic_url
-    |> prepare_url(path)
-    |> HTTP.put(data, [], httpoison_options)
+  """
+  @spec post(binary, list, Keyword.t, Keyword.t) :: HTTP.resp
+  def post(elastic_url, operations, options \\ [], query_params \\ []) do
+    data = for op <- operations, do: [JSON.encode!(op), "\n"]
+    post_raw(elastic_url, data, options, query_params)
   end
 
   @doc """
-  Deprecated: use `post/4` instead.
-  """
-  @spec post_to_iolist(
-          elastic_url :: String.t(),
-          lines :: list,
-          opts :: Keyword.t(),
-          query_params :: Keyword.t()
-        ) :: HTTP.resp()
-  def post_to_iolist(elastic_url, lines, options \\ [], query_params \\ []) do
-    IO.warn(
-      "This function is deprecated and will be removed in future releases; use Elastix.Bulk.post/4 instead."
-    )
+  Send a list of actions and sources, with data already encoded as JSON.
 
-    httpoison_options = Keyword.get(options, :httpoison_options, [])
+  Data should be encoded as described in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html).
 
-    (elastic_url <>
-       make_path(Keyword.get(options, :index), Keyword.get(options, :type), query_params))
-    |> HTTP.put(Enum.map(lines, fn line -> JSON.encode!(line) <> "\n" end), [], httpoison_options)
-  end
+  Options:
+
+  * index: Default index name if actions don't specify one
+  * type: Default type if actions don't specify one.
+    Type is obsolete in newer versions of Elasticsearch.
+    See [removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html)
+  * httpoison_options: options for HTTP call, e.g. setting timeout
 
-  @doc """
-  Same as `post/4` but instead of sending a list of maps you must send raw binary data in
-  the format described in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html).
   """
-  @spec post_raw(
-          elastic_url :: String.t(),
-          raw_data :: String.t(),
-          opts :: Keyword.t(),
-          query_params :: Keyword.t()
-        ) :: HTTP.resp()
+  @spec post_raw(binary, iodata, Keyword.t, Keyword.t) :: HTTP.resp
   def post_raw(elastic_url, raw_data, options \\ [], query_params \\ []) do
-
-    httpoison_options = Keyword.get(options, :httpoison_options, [])
-
-    (elastic_url <>
-       make_path(Keyword.get(options, :index), Keyword.get(options, :type), query_params))
-    |> HTTP.put(raw_data, [], httpoison_options)
+    url = HTTP.make_url(elastic_url, make_path(options[:index], options[:type]), query_params)
+    headers = [{"Content-Type", "application/x-ndjson"}]
+    httpoison_options = options[:httpoison_options] || []
+    HTTP.post(url, raw_data, headers, httpoison_options)
   end
 
-  @doc false
-  def make_path(index_name, type_name, query_params) do
-    path = make_base_path(index_name, type_name)
-
-    case query_params do
-      [] -> path
-      _ -> HTTP.append_query_string(path, query_params)
-    end
+  @doc deprecated: "Use post/4 instead"
+  @spec post_to_iolist(binary, list, Keyword.t, Keyword.t) :: HTTP.resp
+  def post_to_iolist(elastic_url, lines, options \\ [], query_params \\ []) do
+    Logger.warn("This function is deprecated and will be removed in future releases; use Elastix.Bulk.post/4 instead.")
+    httpoison_options = options[:httpoison_options] || []
+    url = HTTP.make_url(elastic_url, make_path(options[:index], options[:type]), query_params)
+    HTTP.post(url, Enum.map(lines, fn line -> JSON.encode!(line) <> "\n" end), [], httpoison_options)
   end
 
-  defp make_base_path(nil, nil), do: "/_bulk"
-  defp make_base_path(index_name, nil), do: "/#{index_name}/_bulk"
-  defp make_base_path(index_name, type_name), do: "/#{index_name}/#{type_name}/_bulk"
+  @doc false
+  # Make path based on index and type options
+  @spec make_path(binary | nil, binary | nil) :: binary
+  def make_path(index, type)
+  def make_path(nil, nil), do: "/_bulk"
+  def make_path(index, nil), do: "/#{index}/_bulk"
+  def make_path(index, type), do: "/#{index}/#{type}/_bulk"
 end