Merge remote-tracking branch 'upstream/master'

.dialyzer_ignore.exs

@@ -0,0 +1,5 @@
+[
+  {"test/support/channel_case.ex", :unknown_function},
+  {"test/support/conn_case.ex", :unknown_function},
+  {"test/support/data_case.ex", :unknown_function}
+]

.github/workflows/elixir.yml

@@ -16,7 +16,7 @@ jobs:
             _build
             .cargo
             deps
-          key: ${{ runner.os }}-deps-2-${{ hashFiles('mix.lock') }}
+          key: ${{ runner.os }}-deps-3-${{ hashFiles('mix.lock') }}
 
       - name: Enable caching
         run: |
@@ -34,18 +34,6 @@ jobs:
       - name: Build and test
         run: docker compose run app run-test
 
-      - name: mix format
-        run: docker compose run app mix format --check-formatted
-
-      - name: Security lint
-        run: |
-          docker compose run app mix sobelow --config
-          docker compose run app mix deps.audit
-
-      - name: Dialyzer
-        run: |
-          docker compose run app mix dialyzer
-
   typos:
     name: 'Check for spelling errors'
     runs-on: ubuntu-latest

docker-compose.yml

@@ -59,15 +59,15 @@ services:
       - '5173:5173'
 
   postgres:
-    image: postgres:16.4-alpine
+    image: postgres:17.2-alpine
     environment:
       - POSTGRES_PASSWORD=postgres
     volumes:
       - postgres_data:/var/lib/postgresql/data
     attach: false
 
   opensearch:
-    image: opensearchproject/opensearch:2.16.0
+    image: opensearchproject/opensearch:2.18.0
     volumes:
       - opensearch_data:/usr/share/opensearch/data
       - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
@@ -82,7 +82,7 @@ services:
     attach: false
 
   files:
-    image: andrewgaul/s3proxy:sha-4976e17
+    image: andrewgaul/s3proxy:sha-3ff077f
     environment:
       - JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3
     volumes:

docker/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM elixir:1.18.0-alpine
+FROM elixir:1.18.1-alpine
 
 ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json
 RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \

docker/app/run-test

@@ -1,10 +1,14 @@
 #!/usr/bin/env sh
+set -e
 
 export MIX_ENV=test
 
 # Always install mix dependencies
 (cd /srv/philomena && mix deps.get)
 
+# Run formatting check
+mix format --check-formatted
+
 # Sleep to allow OpenSearch to finish initializing
 # if it's not done doing whatever it does yet
 echo -n "Waiting for OpenSearch"
@@ -21,4 +25,11 @@ mix ecto.create
 mix ecto.load
 
 # Test the application
-exec mix test
+mix test
+
+# Security lint
+mix sobelow --config
+mix deps.audit
+
+# Static analysis
+exec mix dialyzer

docker/web/Dockerfile

@@ -1,4 +1,4 @@
-FROM openresty/openresty:1.25.3.1-4-alpine
+FROM openresty/openresty:1.27.1.1-alpine
 ARG APP_DIR
 ARG S3_SCHEME
 ARG S3_HOST

lib/philomena/search_indexer.ex

@@ -40,6 +40,16 @@ defmodule Philomena.SearchIndexer do
     Tag => Tags
   }
 
+  @batch_sizes %{
+    Comment => 2048,
+    Filter => 2048,
+    Gallery => 1024,
+    Image => 32,
+    Post => 2048,
+    Report => 128,
+    Tag => 2048
+  }
+
   @doc """
   Recreate the index corresponding to all schemas, and then reindex all of the
   documents within.
@@ -53,11 +63,7 @@ defmodule Philomena.SearchIndexer do
   @spec recreate_reindex_all_destructive! :: :ok
   def recreate_reindex_all_destructive! do
     @schemas
-    |> Task.async_stream(
-      &recreate_reindex_schema_destructive!/1,
-      ordered: false,
-      timeout: :infinity
-    )
+    |> Stream.map(&recreate_reindex_schema_destructive!/1)
     |> Stream.run()
   end
 
@@ -91,11 +97,7 @@ defmodule Philomena.SearchIndexer do
   @spec reindex_all :: :ok
   def reindex_all do
     @schemas
-    |> Task.async_stream(
-      &reindex_schema/1,
-      ordered: false,
-      timeout: :infinity
-    )
+    |> Stream.map(&reindex_schema/1)
     |> Stream.run()
   end
 
@@ -115,12 +117,17 @@ defmodule Philomena.SearchIndexer do
     # Reports currently require handling for their polymorphic nature
     Report
     |> preload([:user, :admin])
-    |> Batch.record_batches()
-    |> Enum.each(fn records ->
-      records
-      |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type])
-      |> Enum.map(&Search.index_document(&1, Report))
-    end)
+    |> Batch.record_batches(batch_size: @batch_sizes[Report])
+    |> Task.async_stream(
+      fn records ->
+        records
+        |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type])
+        |> Enum.map(&Search.index_document(&1, Report))
+      end,
+      timeout: :infinity,
+      max_concurrency: max_concurrency()
+    )
+    |> Stream.run()
   end
 
   def reindex_schema(schema) when schema in @schemas do
@@ -129,6 +136,14 @@ defmodule Philomena.SearchIndexer do
 
     schema
     |> preload(^context.indexing_preloads())
-    |> Search.reindex(schema)
+    |> Search.reindex(schema,
+      batch_size: @batch_sizes[schema],
+      max_concurrency: max_concurrency()
+    )
+  end
+
+  @spec max_concurrency() :: pos_integer()
+  defp max_concurrency do
+    System.schedulers_online()
   end
 end

lib/philomena_media/processors.ex

@@ -91,7 +91,7 @@ defmodule PhilomenaMedia.Processors do
 
   @doc """
   Returns a processor, with the processor being a module capable
-  of processing this content type, or nil.
+  of processing this content type.
 
   The allowed MIME types are:
   - `image/gif`
@@ -111,18 +111,20 @@ defmodule PhilomenaMedia.Processors do
       PhilomenaMedia.Processors.Png
 
       iex> PhilomenaMedia.Processors.processor("application/octet-stream")
-      nil
+      ** (ArgumentError) invalid content type application/octet-stream
 
   """
-  @spec processor(Mime.t()) :: module() | nil
+  @spec processor(Mime.t()) :: module()
   def processor(content_type)
 
   def processor("image/gif"), do: Gif
   def processor("image/jpeg"), do: Jpeg
   def processor("image/png"), do: Png
   def processor("image/svg+xml"), do: Svg
   def processor("video/webm"), do: Webm
-  def processor(_content_type), do: nil
+
+  def processor(content_type),
+    do: raise(ArgumentError, message: "invalid content type #{content_type}")
 
   @doc """
   Takes a MIME type and filtered version list and generates a list of version files to be

lib/philomena_query/search.ex

@@ -201,23 +201,31 @@ defmodule PhilomenaQuery.Search do
   """
   @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok
   def reindex(queryable, module, opts \\ []) do
+    max_concurrency = Keyword.get(opts, :max_concurrency, 1)
     index = @policy.index_for(module)
 
     queryable
-    |> Batch.record_batches(opts)
-    |> Enum.each(fn records ->
-      lines =
-        Enum.flat_map(records, fn record ->
-          doc = index.as_json(record)
-
-          [
-            %{index: %{_index: index.index_name(), _id: doc.id}},
-            doc
-          ]
-        end)
-
-      Api.bulk(@policy.opensearch_url(), lines)
-    end)
+    |> Batch.query_batches(opts)
+    |> Task.async_stream(
+      fn query ->
+        lines =
+          query
+          |> Repo.all()
+          |> Enum.flat_map(fn record ->
+            doc = index.as_json(record)
+
+            [
+              %{index: %{_index: index.index_name(), _id: doc.id}},
+              doc
+            ]
+          end)
+
+        Api.bulk(@policy.opensearch_url(), lines)
+      end,
+      timeout: :infinity,
+      max_concurrency: max_concurrency
+    )
+    |> Stream.run()
   end
 
   @doc ~S"""

lib/philomena_query/search/client.ex

@@ -46,10 +46,10 @@ defmodule PhilomenaQuery.Search.Client do
   end
 
   defp encode_body(body) when is_map(body),
-    do: Jason.encode!(body)
+    do: Jason.encode_to_iodata!(body)
 
   defp encode_body(body) when is_list(body),
-    do: [Enum.map_intersperse(body, "\n", &Jason.encode!(&1)), "\n"]
+    do: [Enum.map_intersperse(body, "\n", &Jason.encode_to_iodata!(&1)), "\n"]
 
   defp encode_options,
     do: [headers: request_headers(), receive_timeout: @receive_timeout]

lib/philomena_web/controllers/activity_controller.ex

@@ -91,8 +91,8 @@ defmodule PhilomenaWeb.ActivityController do
 
     streams =
       Channel
-      |> where([c], c.nsfw == false)
       |> where([c], not is_nil(c.last_fetched_at))
+      |> maybe_show_nsfw_channels(conn.cookies["chan_nsfw"])
       |> order_by(desc: :is_live, asc: :title)
       |> limit(6)
       |> Repo.all()
@@ -151,6 +151,9 @@ defmodule PhilomenaWeb.ActivityController do
     )
   end
 
+  defp maybe_show_nsfw_channels(query, "true"), do: query
+  defp maybe_show_nsfw_channels(query, _falsy), do: where(query, [c], c.nsfw == false)
+
   defp multi_search(images, top_scoring, comments, nil) do
     responses =
       Search.msearch_records(

lib/philomena_web/templates/activity/_channel_strip.html.slime

@@ -8,6 +8,10 @@
       /  => image_tag 'no_avatar_original.svg', width: 32, height: 32, alt: "#{channel.title}'s logo'", class: 'channel-strip__image'
       = @channel.title || @channel.short_name
   .flex__fixed.flex__right
+    = if @channel.nsfw do
+      span title="NSFW"
+        ' 🔞
+    '
     = if @channel.is_live do
       span.channel-strip__state.label.label--narrow.label--success
         ' LIVE NOW

lib/philomena_web/templates/image/_image_container.html.slime

@@ -36,7 +36,7 @@
     - {:video, webm, mp4, hover_text} ->
       .media-box__overlay.js-spoiler-info-overlay
       a href=link title=hover_text
-        video alt=hover_text autoplay="autoplay" muted="muted" loop="loop" playsinline="playsinline"
+        video alt=hover_text autoplay="autoplay" muted="muted" loop="loop" playsinline="playsinline" disablepictureinpicture="disablepictureinpicture"
           source src=webm type="video/webm"
           source src=mp4 type="video/mp4"
         img alt=hover_text
@@ -50,7 +50,7 @@
     - {:filtered_video, hover_text} ->
       .media-box__overlay.js-spoiler-info-overlay
       a href=link title=hover_text
-        video autoplay="autoplay" muted="muted" loop="loop" playsinline="playsinline"
+        video autoplay="autoplay" muted="muted" loop="loop" playsinline="playsinline" disablepictureinpicture="disablepictureinpicture"
         img alt=hover_text
 
     - :not_rendered ->

lib/philomena_web/views/image_view.ex

@@ -264,7 +264,8 @@ defmodule PhilomenaWeb.ImageView do
       height: image.image_height,
       aspect_ratio: image.image_aspect_ratio,
       sha512_hash: image.image_sha512_hash,
-      orig_sha512_hash: image.image_orig_sha512_hash
+      orig_sha512_hash: image.image_orig_sha512_hash,
+      description: image.description
     }
   end
 

mix.exs

@@ -90,7 +90,7 @@ defmodule Philomena.MixProject do
       {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false},
 
       # Static analysis
-      {:dialyxir, "~> 1.2", only: :dev, runtime: false},
+      {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false},
 
       # Fixes for Elixir v1.15+
       {:canary, "~> 1.1",