Merge pull request #341 from philomena-dev/deadweight-elimination-v2

Remove referrer, user_agent, and other unused fields

index/posts.mk

@@ -21,8 +21,8 @@ metadata: post_search_json
 			'body', p.body,
 			'subject', t.title,
 			'ip', p.ip,
-			'user_agent', p.user_agent,
-			'referrer', p.referrer,
+			'user_agent', '',
+			'referrer', '',
 			'fingerprint', p.fingerprint,
 			'topic_position', p.topic_position,
 			'forum', f.short_name,

lib/philomena/artist_links/artist_link.ex

@@ -13,8 +13,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
 
     field :aasm_state, :string, default: "unverified"
     field :uri, :string
-    field :hostname, :string
-    field :path, :string
     field :verification_code, :string
     field :public, :boolean, default: true
     field :next_check_at, :utc_datetime
@@ -35,15 +33,13 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> cast(attrs, [:uri, :public])
     |> put_change(:tag_id, nil)
     |> validate_required([:user, :uri, :public])
-    |> parse_uri()
   end
 
   def edit_changeset(artist_link, attrs, tag) do
     artist_link
     |> cast(attrs, [:uri, :public])
     |> put_change(:tag_id, tag.id)
     |> validate_required([:user, :uri, :public])
-    |> parse_uri()
   end
 
   def creation_changeset(artist_link, attrs, user, tag) do
@@ -55,7 +51,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> validate_required([:tag], message: "must exist")
     |> validate_format(:uri, ~r|\Ahttps?://|)
     |> validate_category()
-    |> parse_uri()
     |> put_verification_code()
     |> put_next_check_at()
     |> unique_constraint([:uri, :tag_id, :user_id],
@@ -95,14 +90,6 @@ defmodule Philomena.ArtistLinks.ArtistLink do
     |> put_change(:aasm_state, "contacted")
   end
 
-  defp parse_uri(changeset) do
-    string_uri = get_field(changeset, :uri) |> to_string()
-    uri = URI.parse(string_uri)
-
-    changeset
-    |> change(hostname: uri.host, path: uri.path)
-  end
-
   defp put_verification_code(changeset) do
     code = :crypto.strong_rand_bytes(5) |> Base.encode16()
     change(changeset, verification_code: "DERPI-LINKVALIDATION-#{code}")

lib/philomena/channels/channel.ex

@@ -12,24 +12,13 @@ defmodule Philomena.Channels.Channel do
 
     field :short_name, :string
     field :title, :string, default: ""
-    field :tags, :string
     field :viewers, :integer, default: 0
     field :nsfw, :boolean, default: false
     field :is_live, :boolean, default: false
     field :last_fetched_at, :utc_datetime
     field :next_check_at, :utc_datetime
     field :last_live_at, :utc_datetime
 
-    field :viewer_minutes_today, :integer, default: 0
-    field :viewer_minutes_thisweek, :integer, default: 0
-    field :viewer_minutes_thismonth, :integer, default: 0
-    field :total_viewer_minutes, :integer, default: 0
-
-    field :banner_image, :string
-    field :channel_image, :string
-    field :remote_stream_id, :integer
-    field :thumbnail_url, :string, default: ""
-
     timestamps(inserted_at: :created_at, type: :utc_datetime)
   end
 

lib/philomena/comments/comment.ex

@@ -14,15 +14,12 @@ defmodule Philomena.Comments.Comment do
     field :body, :string
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :anonymous, :boolean, default: false
     field :hidden_from_users, :boolean, default: false
     field :edit_reason, :string
     field :edited_at, :utc_datetime
     field :deletion_reason, :string, default: ""
     field :destroyed_content, :boolean, default: false
-    field :name_at_post_time, :string
     field :approved, :boolean
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
@@ -35,7 +32,6 @@ defmodule Philomena.Comments.Comment do
     |> validate_required([:body])
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -74,7 +70,4 @@ defmodule Philomena.Comments.Comment do
     change(comment)
     |> put_change(:approved, true)
   end
-
-  defp put_name_at_post_time(changeset, nil), do: changeset
-  defp put_name_at_post_time(changeset, user), do: change(changeset, name_at_post_time: user.name)
 end

lib/philomena/images.ex

@@ -95,11 +95,6 @@ defmodule Philomena.Images do
 
     Multi.new()
     |> Multi.insert(:image, image)
-    |> Multi.run(:name_caches, fn repo, %{image: image} ->
-      image
-      |> Image.cache_changeset()
-      |> repo.update()
-    end)
     |> Multi.run(:added_tag_count, fn repo, %{image: image} ->
       tag_ids = image.added_tags |> Enum.map(& &1.id)
       tags = Tag |> where([t], t.id in ^tag_ids)
@@ -385,8 +380,6 @@ defmodule Philomena.Images do
       updated_at: now,
       ip: attribution[:ip],
       fingerprint: attribution[:fingerprint],
-      user_agent: attribution[:user_agent],
-      referrer: attribution[:referrer],
       added: added
     }
   end
@@ -522,8 +515,6 @@ defmodule Philomena.Images do
       tag_name_cache: tag.name,
       ip: attribution[:ip],
       fingerprint: attribution[:fingerprint],
-      user_agent: attribution[:user_agent],
-      referrer: attribution[:referrer],
       added: added
     }
   end

lib/philomena/images/image.ex

@@ -2,7 +2,6 @@ defmodule Philomena.Images.Image do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query
 
   alias Philomena.ImageIntensities.ImageIntensity
   alias Philomena.ImageVotes.ImageVote
@@ -59,14 +58,11 @@ defmodule Philomena.Images.Image do
     field :image_is_animated, :boolean, source: :is_animated
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :anonymous, :boolean, default: false
     field :score, :integer, default: 0
     field :faves_count, :integer, default: 0
     field :upvotes_count, :integer, default: 0
     field :downvotes_count, :integer, default: 0
-    field :votes_count, :integer, default: 0
     field :source_url, :string
     field :description, :string, default: ""
     field :image_sha512_hash, :string
@@ -88,11 +84,6 @@ defmodule Philomena.Images.Image do
     field :hides_count, :integer, default: 0
     field :approved, :boolean
 
-    # todo: can probably remove these now
-    field :tag_list_cache, :string
-    field :tag_list_plus_alias_cache, :string
-    field :file_name_cache, :string
-
     field :removed_tags, {:array, :any}, default: [], virtual: true
     field :added_tags, {:array, :any}, default: [], virtual: true
     field :removed_sources, {:array, :any}, default: [], virtual: true
@@ -228,7 +219,6 @@ defmodule Philomena.Images.Image do
     |> cast(attrs, [])
     |> TagDiffer.diff_input(old_tags, new_tags, excluded_tags)
     |> TagValidator.validate_tags()
-    |> cache_changeset()
   end
 
   def locked_tags_changeset(image, attrs, locked_tags) do
@@ -345,53 +335,6 @@ defmodule Philomena.Images.Image do
     |> put_change(:first_seen_at, DateTime.utc_now(:second))
   end
 
-  def cache_changeset(image) do
-    changeset = change(image)
-    image = apply_changes(changeset)
-
-    {tag_list_cache, tag_list_plus_alias_cache, file_name_cache} =
-      create_caches(image.id, image.tags)
-
-    changeset
-    |> put_change(:tag_list_cache, tag_list_cache)
-    |> put_change(:tag_list_plus_alias_cache, tag_list_plus_alias_cache)
-    |> put_change(:file_name_cache, file_name_cache)
-  end
-
-  defp create_caches(image_id, tags) do
-    tags = Tag.display_order(tags)
-
-    tag_list_cache =
-      tags
-      |> Enum.map_join(", ", & &1.name)
-
-    tag_ids = tags |> Enum.map(& &1.id)
-
-    aliases =
-      Tag
-      |> where([t], t.aliased_tag_id in ^tag_ids)
-      |> Repo.all()
-
-    tag_list_plus_alias_cache =
-      (tags ++ aliases)
-      |> Tag.display_order()
-      |> Enum.map_join(", ", & &1.name)
-
-    # Truncate filename to 150 characters, making room for the path + filename on Windows
-    # https://stackoverflow.com/questions/265769/maximum-filename-length-in-ntfs-windows-xp-and-windows-vista
-    file_name_slug_fragment =
-      tags
-      |> Enum.map_join("_", & &1.slug)
-      |> String.to_charlist()
-      |> Enum.filter(&(&1 in ?a..?z or &1 in ~c"0123456789_-"))
-      |> List.to_string()
-      |> String.slice(0..150)
-
-    file_name_cache = "#{image_id}__#{file_name_slug_fragment}"
-
-    {tag_list_cache, tag_list_plus_alias_cache, file_name_cache}
-  end
-
   defp create_key do
     Base.encode16(:crypto.strong_rand_bytes(6), case: :lower)
   end

lib/philomena/polls/poll.ex

@@ -3,20 +3,16 @@ defmodule Philomena.Polls.Poll do
   import Ecto.Changeset
 
   alias Philomena.Topics.Topic
-  alias Philomena.Users.User
   alias Philomena.PollOptions.PollOption
 
   schema "polls" do
     belongs_to :topic, Topic
-    belongs_to :deleted_by, User
     has_many :options, PollOption
 
     field :title, :string
     field :vote_method, :string
     field :active_until, PhilomenaQuery.Ecto.RelativeDate
     field :total_votes, :integer, default: 0
-    field :hidden_from_users, :boolean, default: false
-    field :deletion_reason, :string, default: ""
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
   end

lib/philomena/posts/post.ex

@@ -15,15 +15,12 @@ defmodule Philomena.Posts.Post do
     field :edit_reason, :string
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :topic_position, :integer
     field :hidden_from_users, :boolean, default: false
     field :anonymous, :boolean, default: false
     field :edited_at, :utc_datetime
     field :deletion_reason, :string, default: ""
     field :destroyed_content, :boolean, default: false
-    field :name_at_post_time, :string
     field :approved, :boolean, default: false
 
     timestamps(inserted_at: :created_at, type: :utc_datetime)
@@ -47,7 +44,6 @@ defmodule Philomena.Posts.Post do
     |> validate_required([:body])
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -61,7 +57,6 @@ defmodule Philomena.Posts.Post do
     |> validate_length(:body, min: 1, max: 300_000, count: :bytes)
     |> change(attribution)
     |> change(topic_position: 0)
-    |> put_name_at_post_time(attribution[:user])
     |> Approval.maybe_put_approval(attribution[:user])
     |> Approval.maybe_strip_images(attribution[:user])
   end
@@ -90,7 +85,4 @@ defmodule Philomena.Posts.Post do
     change(post)
     |> put_change(:approved, true)
   end
-
-  defp put_name_at_post_time(changeset, nil), do: changeset
-  defp put_name_at_post_time(changeset, user), do: change(changeset, name_at_post_time: user.name)
 end

lib/philomena/posts/search_index.ex

@@ -52,8 +52,8 @@ defmodule Philomena.Posts.SearchIndex do
       author: if(!!post.user and !post.anonymous, do: String.downcase(post.user.name)),
       subject: post.topic.title,
       ip: post.ip |> to_string(),
-      user_agent: post.user_agent,
-      referrer: post.referrer,
+      user_agent: "",
+      referrer: "",
       fingerprint: post.fingerprint,
       topic_position: post.topic_position,
       forum: post.topic.forum.short_name,

lib/philomena/reports.ex

@@ -139,9 +139,7 @@ defmodule Philomena.Reports do
     attributes = %{
       system: true,
       ip: %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32},
-      fingerprint: "ffff",
-      user_agent:
-        "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0"
+      fingerprint: "ffff"
     }
 
     %Report{reportable_type: reportable_type, reportable_id: reportable_id}

lib/philomena/reports/report.ex

@@ -11,7 +11,6 @@ defmodule Philomena.Reports.Report do
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
     field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :reason, :string
     field :state, :string, default: "open"
     field :open, :boolean, default: true
@@ -61,8 +60,9 @@ defmodule Philomena.Reports.Report do
   @doc false
   def creation_changeset(report, attrs, attribution) do
     report
-    |> cast(attrs, [:category, :reason])
+    |> cast(attrs, [:category, :reason, :user_agent])
     |> validate_length(:reason, max: 10_000, count: :bytes)
+    |> validate_length(:user_agent, max: 1000, count: :bytes)
     |> merge_category()
     |> change(attribution)
     |> validate_required([

lib/philomena/roles/role.ex

@@ -4,12 +4,7 @@ defmodule Philomena.Roles.Role do
 
   schema "roles" do
     field :name, :string
-
-    # fixme: rails polymorphic relation
-    field :resource_id, :integer
     field :resource_type, :string
-
-    timestamps(inserted_at: :created_at, type: :utc_datetime)
   end
 
   @doc false

lib/philomena/source_changes/source_change.ex

@@ -8,8 +8,6 @@ defmodule Philomena.SourceChanges.SourceChange do
 
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :value, :string
     field :added, :boolean
 

lib/philomena/tag_changes/tag_change.ex

@@ -9,8 +9,6 @@ defmodule Philomena.TagChanges.TagChange do
 
     field :ip, EctoNetwork.INET
     field :fingerprint, :string
-    field :user_agent, :string, default: ""
-    field :referrer, :string, default: ""
     field :added, :boolean
     field :tag_name_cache, :string, default: ""
 

lib/philomena/users/eraser.ex

@@ -115,9 +115,7 @@ defmodule Philomena.Users.Eraser do
     attribution = [
       user: user,
       ip: @wipe_ip,
-      fingerprint: @wipe_fp,
-      user_agent: "",
-      referrer: ""
+      fingerprint: @wipe_fp
     ]
 
     {:ok, _} = Images.update_sources(source_change.image, attribution, attrs)

lib/philomena/users/user.ex

@@ -109,7 +109,6 @@ defmodule Philomena.Users.User do
     field :watched_tag_list, :string, virtual: true
 
     # Other stuff
-    field :last_donation_at, :utc_datetime
     field :last_renamed_at, :utc_datetime
     field :deleted_at, :utc_datetime
     field :scratchpad, :string