From 8f3f2c981bd8131d289f56baf16ed6ce5f8347b2 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Fri, 19 Jul 2024 02:01:27 -0700 Subject: [PATCH 1/6] feat: support limit param in reverse search --- lib/philomena/duplicate_reports.ex | 4 ++-- .../api/json/search/reverse_controller.ex | 1 + lib/philomena_web/image_reverse.ex | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/philomena/duplicate_reports.ex b/lib/philomena/duplicate_reports.ex index 3e07151ec..d802f3640 100644 --- a/lib/philomena/duplicate_reports.ex +++ b/lib/philomena/duplicate_reports.ex @@ -25,7 +25,7 @@ defmodule Philomena.DuplicateReports do end) end - def duplicates_of(intensities, aspect_ratio, dist \\ 0.25, aspect_dist \\ 0.05) do + def duplicates_of(intensities, aspect_ratio, dist \\ 0.25, aspect_dist \\ 0.05, limit \\ 10) do # for each color channel dist = dist * 3 @@ -39,7 +39,7 @@ defmodule Philomena.DuplicateReports do where: i.image_aspect_ratio >= ^(aspect_ratio - aspect_dist) and i.image_aspect_ratio <= ^(aspect_ratio + aspect_dist), - limit: 10 + limit: ^limit end @doc """ diff --git a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex index e1cf4d7cd..ba94b753e 100644 --- a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex @@ -13,6 +13,7 @@ defmodule PhilomenaWeb.Api.Json.Search.ReverseController do images = image_params |> Map.put("distance", conn.params["distance"]) + |> Map.put("limit", conn.params["limit"]) |> ImageReverse.images() interactions = Interactions.user_interactions(images, user) diff --git a/lib/philomena_web/image_reverse.ex b/lib/philomena_web/image_reverse.ex index 161ebad3c..2ef5e4276 100644 --- a/lib/philomena_web/image_reverse.ex +++ b/lib/philomena_web/image_reverse.ex @@ -18,8 +18,9 @@ defmodule PhilomenaWeb.ImageReverse do {width, height} = analysis.dimensions aspect = width / height dist = normalize_dist(image_params) + limit = parse_limit(image_params) - DuplicateReports.duplicates_of(intensities, aspect, dist, dist) + DuplicateReports.duplicates_of(intensities, aspect, dist, dist, limit) |> preload([:user, :intensity, [:sources, tags: :aliases]]) |> Repo.all() end @@ -60,4 +61,17 @@ defmodule PhilomenaWeb.ImageReverse do 0.0 end end + + defp parse_limit(%{"limit" => limit}) do + limit + |> Integer.parse() + |> case do + {limit, _rest} -> limit + _ -> 10 + end + |> max(1) + |> min(50) + end + + defp parse_limit(_), do: 10 end From 79f508f6031e7ccec54c0a39a8c2ea9449d7f6eb Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 19 Jul 2024 08:50:16 -0400 Subject: [PATCH 2/6] Improve readability of duplicate report frontend parsing / generation --- lib/philomena/duplicate_reports.ex | 9 ++++- lib/philomena_web/image_reverse.ex | 37 +++++++++---------- .../templates/search/reverse/index.html.slime | 2 + 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/lib/philomena/duplicate_reports.ex b/lib/philomena/duplicate_reports.ex index d802f3640..9bf85cdb8 100644 --- a/lib/philomena/duplicate_reports.ex +++ b/lib/philomena/duplicate_reports.ex @@ -15,7 +15,8 @@ defmodule Philomena.DuplicateReports do def generate_reports(source) do source = Repo.preload(source, :intensity) - duplicates_of(source.intensity, source.image_aspect_ratio, 0.2, 0.05) + {source.intensity, source.image_aspect_ratio} + |> find_duplicates(dist: 0.2) |> where([i, _it], i.id != ^source.id) |> Repo.all() |> Enum.map(fn target -> @@ -25,7 +26,11 @@ defmodule Philomena.DuplicateReports do end) end - def duplicates_of(intensities, aspect_ratio, dist \\ 0.25, aspect_dist \\ 0.05, limit \\ 10) do + def find_duplicates({intensities, aspect_ratio}, opts \\ []) do + aspect_dist = Keyword.get(opts, :aspect_dist, 0.05) + limit = Keyword.get(opts, :limit, 10) + dist = Keyword.get(opts, :dist, 0.25) + # for each color channel dist = dist * 3 diff --git a/lib/philomena_web/image_reverse.ex b/lib/philomena_web/image_reverse.ex index 2ef5e4276..e8e32f853 100644 --- a/lib/philomena_web/image_reverse.ex +++ b/lib/philomena_web/image_reverse.ex @@ -17,10 +17,11 @@ defmodule PhilomenaWeb.ImageReverse do {analysis, intensities} -> {width, height} = analysis.dimensions aspect = width / height - dist = normalize_dist(image_params) + dist = parse_dist(image_params) limit = parse_limit(image_params) - DuplicateReports.duplicates_of(intensities, aspect, dist, dist, limit) + {intensities, aspect} + |> DuplicateReports.find_duplicates(dist: dist, aspect_dist: dist, limit: limit) |> preload([:user, :intensity, [:sources, tags: :aliases]]) |> Repo.all() end @@ -43,25 +44,18 @@ defmodule PhilomenaWeb.ImageReverse do # The distance metric is taxicab distance, not Euclidean, # because this is more efficient to index. - defp normalize_dist(%{"distance" => distance}) do + defp parse_dist(%{"distance" => distance}) do distance - |> parse_dist() - |> max(0.01) - |> min(1.0) - end - - defp normalize_dist(_dist), do: 0.25 - - defp parse_dist(dist) do - case Decimal.parse(dist) do - {value, _rest} -> - Decimal.to_float(value) - - _ -> - 0.0 + |> Decimal.parse() + |> case do + {value, _rest} -> Decimal.to_float(value) + _ -> 0.25 end + |> clamp(0.01, 1.0) end + defp parse_dist(_params), do: 0.25 + defp parse_limit(%{"limit" => limit}) do limit |> Integer.parse() @@ -69,9 +63,12 @@ defmodule PhilomenaWeb.ImageReverse do {limit, _rest} -> limit _ -> 10 end - |> max(1) - |> min(50) + |> clamp(1, 50) end - defp parse_limit(_), do: 10 + defp parse_limit(_params), do: 10 + + defp clamp(n, min, _max) when n < min, do: min + defp clamp(n, _min, max) when n > max, do: max + defp clamp(n, _min, _max), do: n end diff --git a/lib/philomena_web/templates/search/reverse/index.html.slime b/lib/philomena_web/templates/search/reverse/index.html.slime index 876be9133..d8b934df0 100644 --- a/lib/philomena_web/templates/search/reverse/index.html.slime +++ b/lib/philomena_web/templates/search/reverse/index.html.slime @@ -28,6 +28,8 @@ h1 Reverse Search br = number_input f, :distance, value: 0.25, min: 0, max: 1, step: 0.01, class: "input" + = hidden_input f, :limit, value: 10 + .field = submit "Reverse Search", class: "button" From e6a63f4c2514624ece1e610a69cd546aeec5636f Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 19 Jul 2024 09:58:19 -0400 Subject: [PATCH 3/6] Bump req --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 4d006ee97..6aff70505 100644 --- a/mix.lock +++ b/mix.lock @@ -67,7 +67,7 @@ "qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"}, "redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, - "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"}, + "req": {:hex, :req, "0.5.1", "90584216d064389a4ff2d4279fe2c11ff6c812ab00fa01a9fb9d15457f65ba70", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "7ea96a1a95388eb0fefa92d89466cdfedba24032794e5c1147d78ec90db7edca"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, "rustler": {:hex, :rustler, "0.33.0", "4a5b0a7a7b0b51549bea49947beff6fae9bc5d5326104dcd4531261e876b5619", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "7c4752728fee59a815ffd20c3429c55b644041f25129b29cdeb5c470b80ec5fd"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, From 7a5d26144ad374ec08681749128dfbd7af3dc616 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 19 Jul 2024 09:29:02 -0400 Subject: [PATCH 4/6] Convert reverse search parsing to changeset-backed form --- lib/philomena/duplicate_reports.ex | 59 +++++++++++++++ .../duplicate_reports/search_query.ex | 59 +++++++++++++++ lib/philomena/duplicate_reports/uploader.ex | 17 +++++ .../api/json/search/reverse_controller.ex | 11 ++- .../controllers/search/reverse_controller.ex | 15 +++- lib/philomena_web/image_reverse.ex | 74 ------------------- .../templates/search/reverse/index.html.slime | 11 ++- 7 files changed, 163 insertions(+), 83 deletions(-) create mode 100644 lib/philomena/duplicate_reports/search_query.ex create mode 100644 lib/philomena/duplicate_reports/uploader.ex delete mode 100644 lib/philomena_web/image_reverse.ex diff --git a/lib/philomena/duplicate_reports.ex b/lib/philomena/duplicate_reports.ex index 9bf85cdb8..5f84c9daf 100644 --- a/lib/philomena/duplicate_reports.ex +++ b/lib/philomena/duplicate_reports.ex @@ -8,6 +8,8 @@ defmodule Philomena.DuplicateReports do alias Philomena.Repo alias Philomena.DuplicateReports.DuplicateReport + alias Philomena.DuplicateReports.SearchQuery + alias Philomena.DuplicateReports.Uploader alias Philomena.ImageIntensities.ImageIntensity alias Philomena.Images.Image alias Philomena.Images @@ -47,6 +49,63 @@ defmodule Philomena.DuplicateReports do limit: ^limit end + @doc """ + Executes the reverse image search query from parameters. + + ## Examples + + iex> execute_search_query(%{"image" => ..., "distance" => "0.25"}) + {:ok, [%Image{...}, ....]} + + iex> execute_search_query(%{"image" => ..., "distance" => "asdf"}) + {:error, %Ecto.Changeset{}} + + """ + def execute_search_query(attrs \\ %{}) do + %SearchQuery{} + |> SearchQuery.changeset(attrs) + |> Uploader.analyze_upload(attrs) + |> Ecto.Changeset.apply_action(:create) + |> case do + {:ok, search_query} -> + intensities = generate_intensities(search_query) + aspect = search_query.image_aspect_ratio + limit = search_query.limit + dist = search_query.distance + + images = + {intensities, aspect} + |> find_duplicates(dist: dist, aspect_dist: dist, limit: limit) + |> preload([:user, :intensity, [:sources, tags: :aliases]]) + |> Repo.all() + + {:ok, images} + + error -> + error + end + end + + defp generate_intensities(search_query) do + analysis = SearchQuery.to_analysis(search_query) + file = search_query.uploaded_image + + PhilomenaMedia.Processors.intensities(analysis, file) + end + + @doc """ + Returns an `%Ecto.Changeset{}` for tracking search query changes. + + ## Examples + + iex> change_search_query(search_query) + %Ecto.Changeset{source: %SearchQuery{}} + + """ + def change_search_query(%SearchQuery{} = search_query) do + SearchQuery.changeset(search_query) + end + @doc """ Gets a single duplicate_report. diff --git a/lib/philomena/duplicate_reports/search_query.ex b/lib/philomena/duplicate_reports/search_query.ex new file mode 100644 index 000000000..bc9220773 --- /dev/null +++ b/lib/philomena/duplicate_reports/search_query.ex @@ -0,0 +1,59 @@ +defmodule Philomena.DuplicateReports.SearchQuery do + use Ecto.Schema + import Ecto.Changeset + + embedded_schema do + field :distance, :float, default: 0.25 + field :limit, :integer, default: 10 + + field :image_width, :integer + field :image_height, :integer + field :image_format, :string + field :image_duration, :float + field :image_mime_type, :string + field :image_is_animated, :boolean + field :image_aspect_ratio, :float + field :uploaded_image, :string, virtual: true + end + + @doc false + def changeset(search_query, attrs \\ %{}) do + search_query + |> cast(attrs, [:distance, :limit]) + |> validate_number(:distance, greater_than_or_equal_to: 0, less_than_or_equal_to: 1) + |> validate_number(:limit, greater_than_or_equal_to: 1, less_than_or_equal_to: 50) + end + + @doc false + def image_changeset(search_query, attrs \\ %{}) do + search_query + |> cast(attrs, [ + :image_width, + :image_height, + :image_format, + :image_duration, + :image_mime_type, + :image_is_animated, + :image_aspect_ratio, + :uploaded_image + ]) + |> validate_number(:image_width, greater_than: 0) + |> validate_number(:image_height, greater_than: 0) + |> validate_inclusion( + :image_mime_type, + ~W(image/gif image/jpeg image/png image/svg+xml video/webm), + message: "(#{attrs["image_mime_type"]}) is invalid" + ) + end + + @doc false + def to_analysis(search_query) do + %PhilomenaMedia.Analyzers.Result{ + animated?: search_query.image_is_animated, + dimensions: {search_query.image_width, search_query.image_height}, + duration: search_query.image_duration, + extension: search_query.image_format, + mime_type: search_query.image_mime_type + } + end +end diff --git a/lib/philomena/duplicate_reports/uploader.ex b/lib/philomena/duplicate_reports/uploader.ex new file mode 100644 index 000000000..41fc49987 --- /dev/null +++ b/lib/philomena/duplicate_reports/uploader.ex @@ -0,0 +1,17 @@ +defmodule Philomena.DuplicateReports.Uploader do + @moduledoc """ + Upload and processing callback logic for SearchQuery images. + """ + + alias Philomena.DuplicateReports.SearchQuery + alias PhilomenaMedia.Uploader + + def analyze_upload(search_query, params) do + Uploader.analyze_upload( + search_query, + "image", + params["image"], + &SearchQuery.image_changeset/2 + ) + end +end diff --git a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex index ba94b753e..1b7a6011a 100644 --- a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.Api.Json.Search.ReverseController do use PhilomenaWeb, :controller - alias PhilomenaWeb.ImageReverse + alias Philomena.DuplicateReports alias Philomena.Interactions plug PhilomenaWeb.ScraperCachePlug @@ -14,7 +14,14 @@ defmodule PhilomenaWeb.Api.Json.Search.ReverseController do image_params |> Map.put("distance", conn.params["distance"]) |> Map.put("limit", conn.params["limit"]) - |> ImageReverse.images() + |> DuplicateReports.execute_search_query() + |> case do + {:ok, images} -> + images + + {:error, _changeset} -> + [] + end interactions = Interactions.user_interactions(images, user) diff --git a/lib/philomena_web/controllers/search/reverse_controller.ex b/lib/philomena_web/controllers/search/reverse_controller.ex index 54c52dacf..967b968ac 100644 --- a/lib/philomena_web/controllers/search/reverse_controller.ex +++ b/lib/philomena_web/controllers/search/reverse_controller.ex @@ -1,7 +1,8 @@ defmodule PhilomenaWeb.Search.ReverseController do use PhilomenaWeb, :controller - alias PhilomenaWeb.ImageReverse + alias Philomena.DuplicateReports.SearchQuery + alias Philomena.DuplicateReports plug PhilomenaWeb.ScraperCachePlug plug PhilomenaWeb.ScraperPlug, params_key: "image", params_name: "image" @@ -12,12 +13,18 @@ defmodule PhilomenaWeb.Search.ReverseController do def create(conn, %{"image" => image_params}) when is_map(image_params) and image_params != %{} do - images = ImageReverse.images(image_params) + case DuplicateReports.execute_search_query(image_params) do + {:ok, images} -> + changeset = DuplicateReports.change_search_query(%SearchQuery{}) + render(conn, "index.html", title: "Reverse Search", images: images, changeset: changeset) - render(conn, "index.html", title: "Reverse Search", images: images) + {:error, changeset} -> + render(conn, "index.html", title: "Reverse Search", images: nil, changeset: changeset) + end end def create(conn, _params) do - render(conn, "index.html", title: "Reverse Search", images: nil) + changeset = DuplicateReports.change_search_query(%SearchQuery{}) + render(conn, "index.html", title: "Reverse Search", images: nil, changeset: changeset) end end diff --git a/lib/philomena_web/image_reverse.ex b/lib/philomena_web/image_reverse.ex deleted file mode 100644 index e8e32f853..000000000 --- a/lib/philomena_web/image_reverse.ex +++ /dev/null @@ -1,74 +0,0 @@ -defmodule PhilomenaWeb.ImageReverse do - alias PhilomenaMedia.Analyzers - alias PhilomenaMedia.Processors - alias Philomena.DuplicateReports - alias Philomena.Repo - import Ecto.Query - - def images(image_params) do - image_params - |> Map.get("image") - |> analyze() - |> intensities() - |> case do - :error -> - [] - - {analysis, intensities} -> - {width, height} = analysis.dimensions - aspect = width / height - dist = parse_dist(image_params) - limit = parse_limit(image_params) - - {intensities, aspect} - |> DuplicateReports.find_duplicates(dist: dist, aspect_dist: dist, limit: limit) - |> preload([:user, :intensity, [:sources, tags: :aliases]]) - |> Repo.all() - end - end - - defp analyze(%Plug.Upload{path: path}) do - case Analyzers.analyze(path) do - {:ok, analysis} -> {analysis, path} - _ -> :error - end - end - - defp analyze(_upload), do: :error - - defp intensities(:error), do: :error - - defp intensities({analysis, path}) do - {analysis, Processors.intensities(analysis, path)} - end - - # The distance metric is taxicab distance, not Euclidean, - # because this is more efficient to index. - defp parse_dist(%{"distance" => distance}) do - distance - |> Decimal.parse() - |> case do - {value, _rest} -> Decimal.to_float(value) - _ -> 0.25 - end - |> clamp(0.01, 1.0) - end - - defp parse_dist(_params), do: 0.25 - - defp parse_limit(%{"limit" => limit}) do - limit - |> Integer.parse() - |> case do - {limit, _rest} -> limit - _ -> 10 - end - |> clamp(1, 50) - end - - defp parse_limit(_params), do: 10 - - defp clamp(n, min, _max) when n < min, do: min - defp clamp(n, _min, max) when n > max, do: max - defp clamp(n, _min, _max), do: n -end diff --git a/lib/philomena_web/templates/search/reverse/index.html.slime b/lib/philomena_web/templates/search/reverse/index.html.slime index d8b934df0..83dd72a73 100644 --- a/lib/philomena_web/templates/search/reverse/index.html.slime +++ b/lib/philomena_web/templates/search/reverse/index.html.slime @@ -1,6 +1,6 @@ h1 Reverse Search -= form_for :image, ~p"/search/reverse", [multipart: true], fn f -> += form_for @changeset, ~p"/search/reverse", [multipart: true, as: :image], fn f -> p ' Basic image similarity search. Finds uploaded images similar to the one ' provided based on simple intensities and uses the median frame of @@ -13,6 +13,10 @@ h1 Reverse Search p Upload a file from your computer, or provide a link to the page containing the image and click Fetch. .field = file_input f, :image, class: "input js-scraper" + = error_tag f, :image + = error_tag f, :image_width + = error_tag f, :image_height + = error_tag f, :image_mime_type .field.field--inline = url_input f, :url, name: "url", class: "input input--wide js-scraper", placeholder: "Link a deviantART page, a Tumblr post, or the image directly" @@ -26,9 +30,10 @@ h1 Reverse Search .field = label f, :distance, "Match distance (suggested values: between 0.2 and 0.5)" br - = number_input f, :distance, value: 0.25, min: 0, max: 1, step: 0.01, class: "input" + = number_input f, :distance, min: 0, max: 1, step: 0.01, class: "input" + = error_tag f, :distance - = hidden_input f, :limit, value: 10 + = error_tag f, :limit .field = submit "Reverse Search", class: "button" From ee57b9d9713842b026b1813837727b6b03105e3f Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 19 Jul 2024 16:35:07 -0400 Subject: [PATCH 5/6] Add more thorough user change eraser --- lib/philomena/users.ex | 15 +++ lib/philomena/users/eraser.ex | 125 ++++++++++++++++++ lib/philomena/workers/user_erase_worker.ex | 11 ++ .../admin/user/erase_controller.ex | 74 +++++++++++ lib/philomena_web/router.ex | 1 + .../templates/admin/user/erase/new.html.slime | 16 +++ .../templates/profile/_admin_block.html.slime | 6 + .../views/admin/user/erase_view.ex | 3 + 8 files changed, 251 insertions(+) create mode 100644 lib/philomena/users/eraser.ex create mode 100644 lib/philomena/workers/user_erase_worker.ex create mode 100644 lib/philomena_web/controllers/admin/user/erase_controller.ex create mode 100644 lib/philomena_web/templates/admin/user/erase/new.html.slime create mode 100644 lib/philomena_web/views/admin/user/erase_view.ex diff --git a/lib/philomena/users.ex b/lib/philomena/users.ex index b971f0c95..013b6173b 100644 --- a/lib/philomena/users.ex +++ b/lib/philomena/users.ex @@ -18,6 +18,7 @@ defmodule Philomena.Users do alias Philomena.Galleries alias Philomena.Reports alias Philomena.Filters + alias Philomena.UserEraseWorker alias Philomena.UserRenameWorker ## Database getters @@ -683,6 +684,20 @@ defmodule Philomena.Users do |> Repo.update() end + def erase_user(%User{} = user, %User{} = moderator) do + # Deactivate to prevent the user from racing these changes + {:ok, user} = deactivate_user(moderator, user) + + # Rename to prevent usage for brand recognition SEO + random_hex = Base.encode16(:crypto.strong_rand_bytes(16), case: :lower) + {:ok, user} = update_user(user, %{name: "deactivated_#{random_hex}"}) + + # Enqueue a background job to perform the rest of the deletion + Exq.enqueue(Exq, "indexing", UserEraseWorker, [user.id, moderator.id]) + + {:ok, user} + end + defp setup_roles(nil), do: nil defp setup_roles(user) do diff --git a/lib/philomena/users/eraser.ex b/lib/philomena/users/eraser.ex new file mode 100644 index 000000000..d584ac77e --- /dev/null +++ b/lib/philomena/users/eraser.ex @@ -0,0 +1,125 @@ +defmodule Philomena.Users.Eraser do + import Ecto.Query + alias Philomena.Repo + + alias Philomena.Bans + alias Philomena.Comments.Comment + alias Philomena.Comments + alias Philomena.Galleries.Gallery + alias Philomena.Galleries + alias Philomena.Posts.Post + alias Philomena.Posts + alias Philomena.Topics.Topic + alias Philomena.Topics + alias Philomena.Images + alias Philomena.SourceChanges.SourceChange + + alias Philomena.Users + + @reason "Site abuse" + @wipe_ip %Postgrex.INET{address: {127, 0, 1, 1}, netmask: 32} + @wipe_fp "ffff" + + def erase_permanently!(user, moderator) do + # Erase avatar + {:ok, user} = Users.remove_avatar(user) + + # Erase "about me" and personal title + {:ok, user} = Users.update_description(user, %{description: "", personal_title: ""}) + + # Delete all forum posts + Post + |> where(user_id: ^user.id) + |> Repo.all() + |> Enum.each(fn post -> + {:ok, post} = Posts.hide_post(post, %{deletion_reason: @reason}, moderator) + {:ok, _post} = Posts.destroy_post(post) + end) + + # Delete all comments + Comment + |> where(user_id: ^user.id) + |> Repo.all() + |> Enum.each(fn comment -> + {:ok, comment} = Comments.hide_comment(comment, %{deletion_reason: @reason}, moderator) + {:ok, _comment} = Comments.destroy_comment(comment) + end) + + # Delete all galleries + Gallery + |> where(creator_id: ^user.id) + |> Repo.all() + |> Enum.each(fn gallery -> + {:ok, _gallery} = Galleries.delete_gallery(gallery) + end) + + # Delete all posted topics + Topic + |> where(user_id: ^user.id) + |> Repo.all() + |> Enum.each(fn topic -> + {:ok, _topic} = Topics.hide_topic(topic, @reason, moderator) + end) + + # Revert all source changes + SourceChange + |> where(user_id: ^user.id) + |> order_by(desc: :created_at) + |> preload(:image) + |> Repo.all() + |> Enum.each(fn source_change -> + if source_change.added do + revert_added_source_change(source_change, user) + else + revert_removed_source_change(source_change, user) + end + end) + + # Delete all source changes + SourceChange + |> where(user_id: ^user.id) + |> Repo.delete_all() + + # Ban the user + {:ok, _ban} = + Bans.create_user( + moderator, + %{ + "user_id" => user.id, + "reason" => @reason, + "valid_until" => "permanent" + } + ) + + # We succeeded + :ok + end + + defp revert_removed_source_change(source_change, user) do + old_sources = %{} + new_sources = %{"0" => %{"source" => source_change.source_url}} + + revert_source_change(source_change, user, old_sources, new_sources) + end + + defp revert_added_source_change(source_change, user) do + old_sources = %{"0" => %{"source" => source_change.source_url}} + new_sources = %{} + + revert_source_change(source_change, user, old_sources, new_sources) + end + + defp revert_source_change(source_change, user, old_sources, new_sources) do + attrs = %{"old_sources" => old_sources, "sources" => new_sources} + + attribution = [ + user: user, + ip: @wipe_ip, + fingerprint: @wipe_fp, + user_agent: "", + referrer: "" + ] + + {:ok, _} = Images.update_sources(source_change.image, attribution, attrs) + end +end diff --git a/lib/philomena/workers/user_erase_worker.ex b/lib/philomena/workers/user_erase_worker.ex new file mode 100644 index 000000000..32c862d37 --- /dev/null +++ b/lib/philomena/workers/user_erase_worker.ex @@ -0,0 +1,11 @@ +defmodule Philomena.UserEraseWorker do + alias Philomena.Users.Eraser + alias Philomena.Users + + def perform(user_id, moderator_id) do + moderator = Users.get_user!(moderator_id) + user = Users.get_user!(user_id) + + Eraser.erase_permanently!(user, moderator) + end +end diff --git a/lib/philomena_web/controllers/admin/user/erase_controller.ex b/lib/philomena_web/controllers/admin/user/erase_controller.ex new file mode 100644 index 000000000..d101db57c --- /dev/null +++ b/lib/philomena_web/controllers/admin/user/erase_controller.ex @@ -0,0 +1,74 @@ +defmodule PhilomenaWeb.Admin.User.EraseController do + use PhilomenaWeb, :controller + + alias Philomena.Users.User + alias Philomena.Users + + plug :verify_authorized + + plug :load_resource, + model: User, + id_name: "user_id", + id_field: "slug", + persisted: true, + preload: [:roles] + + plug :prevent_deleting_privileged_users + plug :prevent_deleting_verified_users + plug :prevent_deleting_old_users + + def new(conn, _params) do + render(conn, "new.html", title: "Erase user") + end + + def create(conn, _params) do + {:ok, user} = Users.erase_user(conn.assigns.user, conn.assigns.current_user) + + conn + |> put_flash(:info, "User erase started") + |> redirect(to: ~p"/profiles/#{user}") + end + + defp verify_authorized(conn, _opts) do + case Canada.Can.can?(conn.assigns.current_user, :index, User) do + true -> conn + _false -> PhilomenaWeb.NotAuthorizedPlug.call(conn) + end + end + + defp prevent_deleting_privileged_users(conn, _opts) do + if conn.assigns.user.role != "user" do + conn + |> put_flash(:error, "Cannot erase a privileged user") + |> redirect(to: ~p"/profiles/#{conn.assigns.user}") + |> Plug.Conn.halt() + else + conn + end + end + + defp prevent_deleting_verified_users(conn, _opts) do + if conn.assigns.user.verified do + conn + |> put_flash(:error, "Cannot erase a verified user") + |> redirect(to: ~p"/profiles/#{conn.assigns.user}") + |> Plug.Conn.halt() + else + conn + end + end + + defp prevent_deleting_old_users(conn, _opts) do + now = DateTime.utc_now(:second) + two_weeks = 1_209_600 + + if DateTime.compare(now, DateTime.add(conn.assigns.user.created_at, two_weeks)) == :gt do + conn + |> put_flash(:error, "Cannot erase a user older than two weeks") + |> redirect(to: ~p"/profiles/#{conn.assigns.user}") + |> Plug.Conn.halt() + else + conn + end + end +end diff --git a/lib/philomena_web/router.ex b/lib/philomena_web/router.ex index 31be10738..be9f48e0b 100644 --- a/lib/philomena_web/router.ex +++ b/lib/philomena_web/router.ex @@ -398,6 +398,7 @@ defmodule PhilomenaWeb.Router do singleton: true resources "/unlock", User.UnlockController, only: [:create], singleton: true + resources "/erase", User.EraseController, only: [:new, :create], singleton: true resources "/api_key", User.ApiKeyController, only: [:delete], singleton: true resources "/downvotes", User.DownvoteController, only: [:delete], singleton: true resources "/votes", User.VoteController, only: [:delete], singleton: true diff --git a/lib/philomena_web/templates/admin/user/erase/new.html.slime b/lib/philomena_web/templates/admin/user/erase/new.html.slime new file mode 100644 index 000000000..643181e74 --- /dev/null +++ b/lib/philomena_web/templates/admin/user/erase/new.html.slime @@ -0,0 +1,16 @@ +h1 + ' Deleting all changes for user + = @user.name + +.block.block--fixed.block--warning + p This is IRREVERSIBLE. + p All user details will be destroyed. + p Are you really sure? + +.field + => button_to "Abort", ~p"/profiles/#{@user}", class: "button" + => button_to "Erase user", ~p"/admin/users/#{@user}/erase", method: "post", class: "button button--state-danger", data: [confirm: "Are you really, really sure?"] + +p + ' This automatically creates user and IP bans but does not create a fingerprint ban. + ' Check to see if one is necessary after erasing. diff --git a/lib/philomena_web/templates/profile/_admin_block.html.slime b/lib/philomena_web/templates/profile/_admin_block.html.slime index 4f8277881..9054a457d 100644 --- a/lib/philomena_web/templates/profile/_admin_block.html.slime +++ b/lib/philomena_web/templates/profile/_admin_block.html.slime @@ -171,6 +171,12 @@ a.label.label--primary.label--block href="#" data-click-toggle=".js-admin__optio i.fa.fa-fw.fa-arrow-down span.admin__button Remove All Downvotes + = if @user.role == "user" do + li + = link to: ~p"/admin/users/#{@user}/erase/new", data: [confirm: "Are you really, really sure?"] do + i.fa.fa-fw.fa-warning + span.admin__button Erase for spam + = if @user.role == "user" and can?(@conn, :revert, Philomena.TagChanges.TagChange) do li = link to: ~p"/tag_changes/full_revert?#{[user_id: @user.id]}", data: [confirm: "Are you really, really sure?", method: "create"] do diff --git a/lib/philomena_web/views/admin/user/erase_view.ex b/lib/philomena_web/views/admin/user/erase_view.ex new file mode 100644 index 000000000..2c4970384 --- /dev/null +++ b/lib/philomena_web/views/admin/user/erase_view.ex @@ -0,0 +1,3 @@ +defmodule PhilomenaWeb.Admin.User.EraseView do + use PhilomenaWeb, :view +end From 335fc0bc56c6f964b98f53731bf9ea89902b197c Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 19 Jul 2024 18:24:06 -0400 Subject: [PATCH 6/6] Remove time limit --- .../controllers/admin/user/erase_controller.ex | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/philomena_web/controllers/admin/user/erase_controller.ex b/lib/philomena_web/controllers/admin/user/erase_controller.ex index d101db57c..b481068e2 100644 --- a/lib/philomena_web/controllers/admin/user/erase_controller.ex +++ b/lib/philomena_web/controllers/admin/user/erase_controller.ex @@ -15,7 +15,6 @@ defmodule PhilomenaWeb.Admin.User.EraseController do plug :prevent_deleting_privileged_users plug :prevent_deleting_verified_users - plug :prevent_deleting_old_users def new(conn, _params) do render(conn, "new.html", title: "Erase user") @@ -57,18 +56,4 @@ defmodule PhilomenaWeb.Admin.User.EraseController do conn end end - - defp prevent_deleting_old_users(conn, _opts) do - now = DateTime.utc_now(:second) - two_weeks = 1_209_600 - - if DateTime.compare(now, DateTime.add(conn.assigns.user.created_at, two_weeks)) == :gt do - conn - |> put_flash(:error, "Cannot erase a user older than two weeks") - |> redirect(to: ~p"/profiles/#{conn.assigns.user}") - |> Plug.Conn.halt() - else - conn - end - end end