Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
liamwhite committed Jun 24, 2024
2 parents cc5c21c + b68dcd4 commit 9f7f83a
Show file tree
Hide file tree
Showing 15 changed files with 228 additions and 70 deletions.
6 changes: 3 additions & 3 deletions config/config.exs
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ config :logger,
config :philomena,
ecto_repos: [Philomena.Repo]

config :elastix,
json_codec: Jason

config :exq,
max_retries: 5,
scheduler_enable: true,
Expand All @@ -37,6 +34,9 @@ config :philomena, PhilomenaWeb.Endpoint,
render_errors: [view: PhilomenaWeb.ErrorView, accepts: ~w(html json)],
pubsub_server: Philomena.PubSub

# Configure only SMTP for mailing, not HTTP
config :swoosh, :api_client, false

# Markdown
config :philomena, Philomena.Native,
crate: "philomena",
Expand Down
4 changes: 0 additions & 4 deletions config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,6 @@ config :philomena, :s3_secondary_options,

config :philomena, :s3_secondary_bucket, System.get_env("ALT_S3_BUCKET")

# Don't bail on OpenSearch's self-signed certificate
config :elastix,
httpoison_options: [ssl: [verify: :verify_none]]

config :ex_aws, http_client: PhilomenaMedia.Req

config :ex_aws, :retries,
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/comments/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Comments.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/filters/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Filters.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/galleries/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Galleries.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/images/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Images.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/posts/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Posts.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/reports/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Reports.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
2 changes: 1 addition & 1 deletion lib/philomena/tags/search_index.ex
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
defmodule Philomena.Tags.SearchIndex do
@behaviour PhilomenaQuery.SearchIndex
@behaviour PhilomenaQuery.Search.Index

@impl true
def index_name do
Expand Down
65 changes: 14 additions & 51 deletions lib/philomena_query/search.ex
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ defmodule PhilomenaQuery.Search do
"""

alias PhilomenaQuery.Batch
alias PhilomenaQuery.Search.Api
alias Philomena.Repo
require Logger
import Ecto.Query
import Elastix.HTTP

# todo: fetch through compile_env?
@policy Philomena.SearchPolicy
Expand Down Expand Up @@ -85,11 +85,7 @@ defmodule PhilomenaQuery.Search do
def create_index!(module) do
index = @policy.index_for(module)

Elastix.Index.create(
@policy.opensearch_url(),
index.index_name(),
index.mapping()
)
Api.create_index(@policy.opensearch_url(), index.index_name(), index.mapping())
end

@doc ~S"""
Expand All @@ -109,7 +105,7 @@ defmodule PhilomenaQuery.Search do
def delete_index!(module) do
index = @policy.index_for(module)

Elastix.Index.delete(@policy.opensearch_url(), index.index_name())
Api.delete_index(@policy.opensearch_url(), index.index_name())
end

@doc ~S"""
Expand All @@ -132,9 +128,7 @@ defmodule PhilomenaQuery.Search do
index_name = index.index_name()
mapping = index.mapping().mappings.properties

Elastix.Mapping.put(@policy.opensearch_url(), index_name, "_doc", %{properties: mapping},
include_type_name: true
)
Api.update_index_mapping(@policy.opensearch_url(), index_name, %{properties: mapping})
end

@doc ~S"""
Expand All @@ -157,13 +151,7 @@ defmodule PhilomenaQuery.Search do
index = @policy.index_for(module)
data = index.as_json(doc)

Elastix.Document.index(
@policy.opensearch_url(),
index.index_name(),
"_doc",
data.id,
data
)
Api.index_document(@policy.opensearch_url(), index.index_name(), data, data.id)
end

@doc ~S"""
Expand All @@ -186,12 +174,7 @@ defmodule PhilomenaQuery.Search do
def delete_document(id, module) do
index = @policy.index_for(module)

Elastix.Document.delete(
@policy.opensearch_url(),
index.index_name(),
"_doc",
id
)
Api.delete_document(@policy.opensearch_url(), index.index_name(), id)
end

@doc """
Expand Down Expand Up @@ -231,12 +214,7 @@ defmodule PhilomenaQuery.Search do
]
end)

Elastix.Bulk.post(
@policy.opensearch_url(),
lines,
index: index.index_name(),
httpoison_options: [timeout: 30_000]
)
Api.bulk(@policy.opensearch_url(), lines)
end)
end

Expand Down Expand Up @@ -272,11 +250,6 @@ defmodule PhilomenaQuery.Search do
def update_by_query(module, query_body, set_replacements, replacements) do
index = @policy.index_for(module)

url =
@policy.opensearch_url()
|> prepare_url([index.index_name(), "_update_by_query"])
|> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"})

# "Painless" scripting language
script = """
// Replace values in "sets" (arrays in the source document)
Expand Down Expand Up @@ -320,7 +293,7 @@ defmodule PhilomenaQuery.Search do
"""

body =
Jason.encode!(%{
%{
script: %{
source: script,
params: %{
Expand All @@ -329,9 +302,9 @@ defmodule PhilomenaQuery.Search do
}
},
query: query_body
})
}

{:ok, %{status_code: 200}} = Elastix.HTTP.post(url, body)
Api.update_by_query(@policy.opensearch_url(), index.index_name(), body)
end

@doc ~S"""
Expand Down Expand Up @@ -360,13 +333,8 @@ defmodule PhilomenaQuery.Search do
def search(module, query_body) do
index = @policy.index_for(module)

{:ok, %{body: results, status_code: 200}} =
Elastix.Search.search(
@policy.opensearch_url(),
index.index_name(),
[],
query_body
)
{:ok, %{body: results, status: 200}} =
Api.search(@policy.opensearch_url(), index.index_name(), query_body)

results
end
Expand Down Expand Up @@ -401,13 +369,8 @@ defmodule PhilomenaQuery.Search do
]
end)

{:ok, %{body: results, status_code: 200}} =
Elastix.Search.search(
@policy.opensearch_url(),
"_all",
[],
msearch_body
)
{:ok, %{body: results, status: 200}} =
Api.msearch(@policy.opensearch_url(), msearch_body)

results["responses"]
end
Expand Down
141 changes: 141 additions & 0 deletions lib/philomena_query/search/api.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
defmodule PhilomenaQuery.Search.Api do
@moduledoc """
Interaction with OpenSearch API by endpoint name.
See https://opensearch.org/docs/latest/api-reference for a complete reference.
"""

alias PhilomenaQuery.Search.Client

@type server_url :: String.t()
@type index_name :: String.t()

@type properties :: map()
@type mapping :: map()
@type document :: map()
@type document_id :: integer()

@doc """
Create the index named `name` with the given `mapping`.
https://opensearch.org/docs/latest/api-reference/index-apis/create-index/
"""
@spec create_index(server_url(), index_name(), mapping()) :: Client.result()
def create_index(url, name, mapping) do
url
|> prepare_url([name])
|> Client.put(mapping)
end

@doc """
Delete the index named `name`.
https://opensearch.org/docs/latest/api-reference/index-apis/delete-index/
"""
@spec delete_index(server_url(), index_name()) :: Client.result()
def delete_index(url, name) do
url
|> prepare_url([name])
|> Client.delete()
end

@doc """
Update the index named `name` with the given `properties`.
https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/
"""
@spec update_index_mapping(server_url(), index_name(), properties()) :: Client.result()
def update_index_mapping(url, name, properties) do
url
|> prepare_url([name, "_mapping"])
|> Client.put(properties)
end

@doc """
Index `document` in the index named `name` with integer id `id`.
https://opensearch.org/docs/latest/api-reference/document-apis/index-document/
"""
@spec index_document(server_url(), index_name(), document(), document_id()) :: Client.result()
def index_document(url, name, document, id) do
url
|> prepare_url([name, "_doc", Integer.to_string(id)])
|> Client.put(document)
end

@doc """
Remove document in the index named `name` with integer id `id`.
https://opensearch.org/docs/latest/api-reference/document-apis/delete-document/
"""
@spec delete_document(server_url(), index_name(), document_id()) :: Client.result()
def delete_document(url, name, id) do
url
|> prepare_url([name, "_doc", Integer.to_string(id)])
|> Client.delete()
end

@doc """
Bulk operation.
https://opensearch.org/docs/latest/api-reference/document-apis/bulk/
"""
@spec bulk(server_url(), list()) :: Client.result()
def bulk(url, lines) do
url
|> prepare_url(["_bulk"])
|> Client.post(lines)
end

@doc """
Asynchronous scripted updates.
Sets `conflicts` to `proceed` and `wait_for_completion` to `false`.
https://opensearch.org/docs/latest/api-reference/document-apis/update-by-query/
"""
@spec update_by_query(server_url(), index_name(), map()) :: Client.result()
def update_by_query(url, name, body) do
url
|> prepare_url([name, "_update_by_query"])
|> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"})
|> Client.post(body)
end

@doc """
Search for documents in index named `name` with `query`.
https://opensearch.org/docs/latest/api-reference/search/
"""
@spec search(server_url(), index_name(), map()) :: Client.result()
def search(url, name, body) do
url
|> prepare_url([name, "_search"])
|> Client.get(body)
end

@doc """
Search for documents in all indices with specified `lines`.
https://opensearch.org/docs/latest/api-reference/multi-search/
"""
@spec msearch(server_url(), list()) :: Client.result()
def msearch(url, lines) do
url
|> prepare_url(["_msearch"])
|> Client.get(lines)
end

@spec prepare_url(String.t(), [String.t()]) :: String.t()
defp prepare_url(url, parts) when is_list(parts) do
# Combine path generated by the parts with the main URL
url
|> URI.merge(Path.join(parts))
|> to_string()
end

@spec append_query_string(String.t(), map()) :: String.t()
defp append_query_string(url, params) do
url <> "?" <> URI.encode_query(params)
end
end
Loading

0 comments on commit 9f7f83a

Please sign in to comment.