exiftool: make stripped tags configurable

Originally part of a larger commit of
https://akkoma.dev/AkkomaGang/akkoma/pulls/241
but just the renaming part was cherry picked and
adapted to reflect now all metadata being stripped
rather than just GPS data.
(i.e. rename to StripMetadata instead of StripLocation)

Cherry-picked-by: Oneric

CHANGELOG.md

@@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Verified support for elixir 1.16
 
 ## Changed
+- Uploadfilter `Pleroma.Upload.Filter.Exiftool` was replaced by `Pleroma.Upload.Filter.Exiftool.StripMetadata`;
+  the latter strips all non-essential metadata by default but can be configured.
+  To regains the old behaviour of only stripping gps data set `purge: ["gps:all"]`.
 
 ## Fixed
 - Issue preventing fetching anything from IPv6-only instances

config/description.exs

@@ -222,6 +222,26 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Upload.Filter.Exiftool.StripMetadata,
+    type: :group,
+    description: "Strip specified metadata from image uploads",
+    children: [
+      %{
+        key: :purge,
+        description: "Metadata fields or groups to strip",
+        type: {:list, :string},
+        suggestions: ["all", "CommonIFD0"]
+      },
+      %{
+        key: :preserve,
+        description: "Metadata fields or groups to preserve (takes precedence over stripping)",
+        type: {:list, :string},
+        suggestions: ["ColorSpaces", "Orientation"]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,

docs/docs/administration/CLI_tasks/instance.md

@@ -37,7 +37,7 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
 - `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
 - `--listen-port <port>` - the port the app should listen to, defaults to 4000
-- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
+- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of metadata when possible
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
 - `--skip-release-env` - skip generation the release environment file

docs/docs/configuration/cheatsheet.md

@@ -654,10 +654,12 @@ This filter replaces the declared filename (not the path) of an upload.
 
 * `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
 
-#### Pleroma.Upload.Filter.Exiftool
+#### Pleroma.Upload.Filter.Exiftool.StripMetadata
 
-This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.
+This filter strips metadata with Exiftool leaving color profiles and orientation intact.
 
+* `purge`: List of Exiftool tag names or tag group names to purge
+* `preserve`: List of Exiftool tag names or tag group names to preserve even if they occur in the purge list
 No specific configuration.
 
 #### Pleroma.Upload.Filter.OnlyMedia

docs/docs/installation/optional/media_graphics_packages.md

@@ -29,4 +29,4 @@ It is required for the following Akkoma features:
 `exiftool` is media files metadata reader/writer.
 
 It is required for the following Akkoma features:
-  * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`)
+  * `Pleroma.Upload.Filters.Exiftool.StripMetadata` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`)

lib/mix/tasks/pleroma/instance.ex

@@ -171,10 +171,10 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       {strip_uploads_message, strip_uploads_default} =
         if Pleroma.Utils.command_available?("exiftool") do
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
+          {"Do you want to strip metadata from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
            "y"}
         else
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+          {"Do you want to strip metadata from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
            "n"}
         end
 
@@ -307,7 +307,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip do
-        [Pleroma.Upload.Filter.Exiftool]
+        [Pleroma.Upload.Filter.Exiftool.StripMetadata]
       else
         []
       end

lib/pleroma/application_requirements.ex

@@ -164,7 +164,7 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.StripMetadata, "exiftool"),
       check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
       check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
       check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),

lib/pleroma/config/deprecation_warnings.ex

@@ -22,6 +22,43 @@ defmodule Pleroma.Config.DeprecationWarnings do
      "\n* `config :pleroma, :instance, :quarantined_instances` is now covered by `:pleroma, :mrf_simple, :reject`"}
   ]
 
+  def check_exiftool_filter do
+    filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
+
+    if Pleroma.Upload.Filter.Exiftool in filters do
+      Logger.warning("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using Exiftool as a filter instead of Exiftool.StripMetadata. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool]
+      ```
+
+      Is now
+
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool.StripMetadata]
+      ```
+      """)
+
+      new_config =
+        filters
+        |> Enum.map(fn
+          Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripMetadata
+          filter -> filter
+        end)
+
+      Config.put([Pleroma.Upload, :filters], new_config)
+
+      :error
+    else
+      :ok
+    end
+  end
+
   def check_simple_policy_tuples do
     has_strings =
       Config.get([:mrf_simple])
@@ -184,7 +221,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
       check_simple_policy_tuples(),
       check_http_adapter(),
       check_uploader_base_url_set(),
-      check_uploader_base_url_is_not_base_domain()
+      check_uploader_base_url_is_not_base_domain(),
+      check_exiftool_filter()
     ]
     |> Enum.reduce(:ok, fn
       :ok, :ok -> :ok

lib/pleroma/upload/filter/exiftool/strip_metadata.ex

@@ -2,13 +2,18 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Upload.Filter.Exiftool do
+defmodule Pleroma.Upload.Filter.Exiftool.StripMetadata do
   @moduledoc """
-  Strips GPS related EXIF tags and overwrites the file in place.
+  Tries to strip all image metadata but colorspace and orientation overwriting the file in place.
   Also strips or replaces filesystem metadata e.g., timestamps.
   """
   @behaviour Pleroma.Upload.Filter
 
+  alias Pleroma.Config
+
+  @purge_default ["all", "CommonIFD0"]
+  @preserve_default ["ColorSpaceTags", "Orientation"]
+
   @spec filter(Pleroma.Upload.t()) :: {:ok, :noop} | {:ok, :filtered} | {:error, String.t()}
 
   # Formats not compatible with exiftool at this time
@@ -16,12 +21,23 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   def filter(%Pleroma.Upload{content_type: "image/svg+xml"}), do: {:ok, :noop}
 
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
+    purge_args =
+      Config.get([__MODULE__, :purge], @purge_default)
+      |> Enum.map(fn mgroup -> "-" <> mgroup <> "=" end)
+
+    preserve_args =
+      Config.get([__MODULE__, :preserve], @preserve_default)
+      |> Enum.map(fn mgroup -> "-" <> mgroup end)
+      |> then(fn
+        # If -TagsFromFile is not followed by tag selectors, it will copy most available tags
+        [] -> []
+        args -> ["-TagsFromFile", "@" | args]
+      end)
+
+    args = ["-ignoreMinorErrors", "-overwrite_original" | purge_args] ++ preserve_args ++ [file]
+
     try do
-      case System.cmd(
-             "exiftool",
-             ["-ignoreMinorErrors", "-overwrite_original", "-gps:all=", file],
-             parallelism: true
-           ) do
+      case System.cmd("exiftool", args, parallelism: true) do
         {_response, 0} -> {:ok, :filtered}
         {error, 1} -> {:error, error}
       end

priv/repo/migrations/20240418190000_upload_filter_exiftool_to_exiftool_strip_metadata.exs

@@ -0,0 +1,37 @@
+defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExiftoolStripMetadata do
+  use Ecto.Migration
+
+  alias Pleroma.ConfigDB
+
+  def up,
+    do:
+      ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload})
+      |> update_filtername(
+        Pleroma.Upload.Filter.Exiftool,
+        Pleroma.Upload.Filter.Exiftool.StripMetadata
+      )
+
+  def down,
+    do:
+      ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload})
+      |> update_filtername(
+        Pleroma.Upload.Filter.Exiftool.StripMetadata,
+        Pleroma.Upload.Filter.Exiftool
+      )
+
+  defp update_filtername(%{value: value}, from_filtername, to_filtername) do
+    new_value =
+      value
+      |> Keyword.update(:filters, [], fn filters ->
+        filters
+        |> Enum.map(fn
+          ^from_filtername -> to_filtername
+          filter -> filter
+        end)
+      end)
+
+    ConfigDB.update_or_create(%{group: :pleroma, key: Pleroma.Upload, value: new_value})
+  end
+
+  defp update_filtername(_, _, _), do: nil
+end

test/mix/tasks/pleroma/instance_test.exs

@@ -91,7 +91,7 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
     assert generated_config =~ "password: \"dbpass\""
     assert generated_config =~ "configurable_from_database: true"
     assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
-    assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]"
+    assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool.StripMetadata]"
     assert generated_config =~ "base_url: \"https://media.pleroma.social/media\""
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))

test/pleroma/config/deprecation_warnings_test.exs

@@ -11,6 +11,62 @@ defmodule Pleroma.Config.DeprecationWarningsTest do
   alias Pleroma.Config
   alias Pleroma.Config.DeprecationWarnings
 
+  describe "filter exiftool" do
+    test "gives warning when still used" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [Pleroma.Upload.Filter.Exiftool]
+      )
+
+      assert capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) =~
+               """
+               !!!DEPRECATION WARNING!!!
+               Your config is using Exiftool as a filter instead of Exiftool.StripMetadata. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+
+               ```
+               config :pleroma, Pleroma.Upload,
+                 filters: [Pleroma.Upload.Filter.Exiftool]
+               ```
+
+               Is now
+
+
+               ```
+               config :pleroma, Pleroma.Upload,
+                 filters: [Pleroma.Upload.Filter.Exiftool.StripMetadata]
+               ```
+               """
+    end
+
+    test "changes setting to exiftool strip metadata" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [Pleroma.Upload.Filter.Exiftool, Pleroma.Upload.Filter.Exiftool.ReadDescription]
+      )
+
+      expected_config = [
+        Pleroma.Upload.Filter.Exiftool.StripMetadata,
+        Pleroma.Upload.Filter.Exiftool.ReadDescription
+      ]
+
+      capture_log(fn -> DeprecationWarnings.warn() end)
+
+      assert Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) == expected_config
+    end
+
+    test "doesn't give a warning with correct config" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [
+          Pleroma.Upload.Filter.Exiftool.StripMetadata,
+          Pleroma.Upload.Filter.Exiftool.ReadDescription
+        ]
+      )
+
+      assert capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) == ""
+    end
+  end
+
   describe "simple policy tuples" do
     test "gives warning when there are still strings" do
       clear_config([:mrf_simple],

test/pleroma/upload/filter/exiftool/strip_metadata_test.exs

@@ -0,0 +1,135 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool.StripMetadataTest do
+  use Pleroma.DataCase
+  alias Pleroma.Upload.Filter
+
+  test "exiftool strip metadata strips GPS etc but preserves Orientation and ColorSpace by default" do
+    assert Pleroma.Utils.command_available?("exiftool")
+
+    tmpfile = "test/fixtures/DSCN0010_#{__MODULE__}_01.jpg"
+
+    File.cp!(
+      "test/fixtures/DSCN0010.jpg",
+      tmpfile
+    )
+
+    upload = %Pleroma.Upload{
+      name: "image_with_GPS_data.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/DSCN0010.jpg"),
+      tempfile: Path.absname(tmpfile)
+    }
+
+    assert Filter.Exiftool.StripMetadata.filter(upload) == {:ok, :filtered}
+
+    exif_original = read_exif("test/fixtures/DSCN0010.jpg")
+    exif_filtered = read_exif(tmpfile)
+
+    refute exif_original == exif_filtered
+    assert String.match?(exif_original, ~r/GPS/)
+    refute String.match?(exif_filtered, ~r/GPS/)
+    assert String.match?(exif_original, ~r/Camera Model Name/)
+    refute String.match?(exif_filtered, ~r/Camera Model Name/)
+    assert String.match?(exif_original, ~r/Orientation/)
+    assert String.match?(exif_filtered, ~r/Orientation/)
+    assert String.match?(exif_original, ~r/Color Space/)
+    assert String.match?(exif_filtered, ~r/Color Space/)
+  end
+
+  # this is a nonsensical configuration, but it shouldn't explode
+  test "exiftool strip metadata is a noop with empty purge list" do
+    assert Pleroma.Utils.command_available?("exiftool")
+    clear_config([Pleroma.Upload.Filter.Exiftool.StripMetadata, :purge], [])
+
+    tmpfile = "test/fixtures/DSCN0010_#{__MODULE__}_02.jpg"
+
+    File.cp!(
+      "test/fixtures/DSCN0010.jpg",
+      tmpfile
+    )
+
+    upload = %Pleroma.Upload{
+      name: "image_with_GPS_data.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/DSCN0010.jpg"),
+      tempfile: Path.absname(tmpfile)
+    }
+
+    assert Filter.Exiftool.StripMetadata.filter(upload) == {:ok, :filtered}
+
+    exif_original = read_exif("test/fixtures/DSCN0010.jpg")
+    exif_filtered = read_exif(tmpfile)
+
+    assert exif_original == exif_filtered
+  end
+
+  test "exiftool strip metadata works with empty preserve list" do
+    assert Pleroma.Utils.command_available?("exiftool")
+    clear_config([Pleroma.Upload.Filter.Exiftool.StripMetadata, :preserve], [])
+
+    tmpfile = "test/fixtures/DSCN0010_#{__MODULE__}_03.jpg"
+
+    File.cp!(
+      "test/fixtures/DSCN0010.jpg",
+      tmpfile
+    )
+
+    upload = %Pleroma.Upload{
+      name: "image_with_GPS_data.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/DSCN0010.jpg"),
+      tempfile: Path.absname(tmpfile)
+    }
+
+    write_exif(["-ImageDescription=Trees and Houses", "-Orientation=1", tmpfile])
+    exif_extended = read_exif(tmpfile)
+    assert String.match?(exif_extended, ~r/Image Description[ \t]*:[ \t]*Trees and Houses/)
+    assert String.match?(exif_extended, ~r/Orientation/)
+
+    assert Filter.Exiftool.StripMetadata.filter(upload) == {:ok, :filtered}
+
+    exif_original = read_exif("test/fixtures/DSCN0010.jpg")
+    exif_filtered = read_exif(tmpfile)
+
+    refute exif_original == exif_filtered
+    refute exif_extended == exif_filtered
+    assert String.match?(exif_original, ~r/GPS/)
+    refute String.match?(exif_filtered, ~r/GPS/)
+    refute String.match?(exif_filtered, ~r/Image Description/)
+    refute String.match?(exif_filtered, ~r/Orientation/)
+  end
+
+  test "verify webp files are skipped" do
+    upload = %Pleroma.Upload{
+      name: "sample.webp",
+      content_type: "image/webp"
+    }
+
+    assert Filter.Exiftool.StripMetadata.filter(upload) == {:ok, :noop}
+  end
+
+  defp read_exif(file) do
+    # time and file path tags cause mismatches even for byte-identical files
+    {exif_data, 0} =
+      System.cmd("exiftool", [
+        "-x",
+        "Time:All",
+        "-x",
+        "Directory",
+        "-x",
+        "FileName",
+        "-x",
+        "FileSize",
+        file
+      ])
+
+    exif_data
+  end
+
+  defp write_exif(args) do
+    {_response, 0} = System.cmd("exiftool", ["-ignoreMinorErrors", "-overwrite_original" | args])
+  end
+end

test/pleroma/upload/filter/exiftool_test.exs

@@ -1,42 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Upload.Filter.ExiftoolTest do
-  use Pleroma.DataCase
-  alias Pleroma.Upload.Filter
-
-  test "apply exiftool filter" do
-    assert Pleroma.Utils.command_available?("exiftool")
-
-    File.cp!(
-      "test/fixtures/DSCN0010.jpg",
-      "test/fixtures/DSCN0010_tmp.jpg"
-    )
-
-    upload = %Pleroma.Upload{
-      name: "image_with_GPS_data.jpg",
-      content_type: "image/jpeg",
-      path: Path.absname("test/fixtures/DSCN0010.jpg"),
-      tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
-    }
-
-    assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
-
-    {exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
-    {exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
-
-    refute exif_original == exif_filtered
-    assert String.match?(exif_original, ~r/GPS/)
-    refute String.match?(exif_filtered, ~r/GPS/)
-  end
-
-  test "verify webp files are skipped" do
-    upload = %Pleroma.Upload{
-      name: "sample.webp",
-      content_type: "image/webp"
-    }
-
-    assert Filter.Exiftool.filter(upload) == {:ok, :noop}
-  end
-end