From 0fbe386839e6f39f877c770cf73587c4421f632d Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Mon, 22 Jan 2024 19:03:06 -0500 Subject: [PATCH 1/6] Add checksum header when has is not md5 --- lib/ex_aws/s3.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ex_aws/s3.ex b/lib/ex_aws/s3.ex index ba78d8f..fd527e7 100644 --- a/lib/ex_aws/s3.ex +++ b/lib/ex_aws/s3.ex @@ -1014,7 +1014,8 @@ defmodule ExAws.S3 do # Supported hash algorithms: # https://www.erlang.org/doc/man/crypto.html#type-hash_algorithm @spec hash_header(atom()) :: binary() - defp hash_header(alg) when is_atom(alg), do: "content-#{to_string(alg)}" + defp hash_header(:md5), do: "content-md5" + defp hash_header(alg) when is_atom(alg), do: "x-amz-checksum-#{to_string(alg)}" @spec pair_tuple_to_map({term(), term()}) :: map() defp pair_tuple_to_map(tuple), do: Map.new([tuple]) From 7918af1d00725c1c2a41d6f1104d08d3c3df29d0 Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Tue, 23 Jan 2024 13:56:49 -0500 Subject: [PATCH 2/6] Add tests for content hash headers --- lib/ex_aws/s3.ex | 2 +- test/lib/s3_test.exs | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/ex_aws/s3.ex b/lib/ex_aws/s3.ex index fd527e7..35463a2 100644 --- a/lib/ex_aws/s3.ex +++ b/lib/ex_aws/s3.ex @@ -1006,7 +1006,7 @@ defmodule ExAws.S3 do {hash_header(alg), :crypto.hash(alg, content) |> Base.encode64()} end - @spec get_hash_config() :: :md5 + @spec get_hash_config() :: atom() defp get_hash_config() do Application.get_env(:ex_aws_s3, :content_hash_algorithm) || :md5 end diff --git a/test/lib/s3_test.exs b/test/lib/s3_test.exs index b6ed8b8..ec09a44 100644 --- a/test/lib/s3_test.exs +++ b/test/lib/s3_test.exs @@ -1,5 +1,6 @@ defmodule ExAws.S3Test do - use ExUnit.Case, async: true + # Setting async: false since we're modifying Application env + use ExUnit.Case, async: false alias ExAws.{S3, Operation} test "#list_objects" do @@ -337,6 +338,28 @@ defmodule ExAws.S3Test do ]) end + test "#delete_multiple_objects sha256" do + set_content_hash_algorithm() + + expected = %Operation.S3{ + body: + "foobarv1special characters: '"&<> ", + bucket: "bucket", + path: "/?delete", + headers: %{"x-amz-checksum-sha256" => "Hsce+MZp64Uy8CwyResJ3y13YGw6jDAAdVGFmFPKPSs="}, + http_method: :post + } + + assert expected == + S3.delete_multiple_objects("bucket", [ + "foo", + {"bar", "v1"}, + "special characters: '\"&<>\r\n" + ]) + + unset_content_hash_algorithm() + end + test "#post_object_restore" do expected = %Operation.S3{ body: @@ -697,6 +720,25 @@ defmodule ExAws.S3Test do expected = %{"content-md5" => content_hash} assert ExAws.S3.calculate_content_header("hello world") === expected end + + test "SHA256" do + set_content_hash_algorithm() + + body = "hello world" + content_hash = :crypto.hash(:sha256, body) |> Base.encode64() + expected = %{"x-amz-checksum-sha256" => content_hash} + assert ExAws.S3.calculate_content_header("hello world") === expected + + unset_content_hash_algorithm() + end + end + + defp set_content_hash_algorithm() do + Application.put_env(:ex_aws_s3, :content_hash_algorithm, :sha256) + end + + defp unset_content_hash_algorithm() do + Application.delete_env(:ex_aws_s3, :content_hash_algorithm) end @spec assert_pre_signed_url( From a076221cd2fa1c15d5a245a0ac2aa8e2f0ff7e53 Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Tue, 23 Jan 2024 15:58:05 -0500 Subject: [PATCH 3/6] Update types and tests --- lib/ex_aws/s3.ex | 13 +++++++++++-- test/lib/s3_test.exs | 45 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/lib/ex_aws/s3.ex b/lib/ex_aws/s3.ex index 35463a2..053831b 100644 --- a/lib/ex_aws/s3.ex +++ b/lib/ex_aws/s3.ex @@ -79,6 +79,14 @@ defmodule ExAws.S3 do @type amz_meta_opts :: [{atom, binary} | {binary, binary}, ...] + @typedoc """ + The hashing algorithms that S3 supports for content integrity. + Erlang only supports sha1 and sha256. + + https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + """ + @type hash_algorithm :: :crc32 | :crc32c | :sha1 | :sha256 + ## Buckets ############# @doc "List buckets" @@ -1011,10 +1019,11 @@ defmodule ExAws.S3 do Application.get_env(:ex_aws_s3, :content_hash_algorithm) || :md5 end - # Supported hash algorithms: + # Supported erlang hash algorithms: # https://www.erlang.org/doc/man/crypto.html#type-hash_algorithm - @spec hash_header(atom()) :: binary() + @spec hash_header(hash_algorithm()) :: binary() defp hash_header(:md5), do: "content-md5" + defp hash_header(:sha), do: "x-amz-checksum-sha1" defp hash_header(alg) when is_atom(alg), do: "x-amz-checksum-#{to_string(alg)}" @spec pair_tuple_to_map({term(), term()}) :: map() diff --git a/test/lib/s3_test.exs b/test/lib/s3_test.exs index ec09a44..d37a024 100644 --- a/test/lib/s3_test.exs +++ b/test/lib/s3_test.exs @@ -338,8 +338,30 @@ defmodule ExAws.S3Test do ]) end + test "#delete_multiple_objects sha1" do + set_content_hash_algorithm(:sha) + + expected = %Operation.S3{ + body: + "foobarv1special characters: '"&<> ", + bucket: "bucket", + path: "/?delete", + headers: %{"x-amz-checksum-sha1" => "1uLbLBmwtufR1/csrQPCAONFeKU="}, + http_method: :post + } + + assert expected == + S3.delete_multiple_objects("bucket", [ + "foo", + {"bar", "v1"}, + "special characters: '\"&<>\r\n" + ]) + + on_exit(&unset_content_hash_algorithm/0) + end + test "#delete_multiple_objects sha256" do - set_content_hash_algorithm() + set_content_hash_algorithm(:sha256) expected = %Operation.S3{ body: @@ -357,7 +379,7 @@ defmodule ExAws.S3Test do "special characters: '\"&<>\r\n" ]) - unset_content_hash_algorithm() + on_exit(&unset_content_hash_algorithm/0) end test "#post_object_restore" do @@ -721,20 +743,31 @@ defmodule ExAws.S3Test do assert ExAws.S3.calculate_content_header("hello world") === expected end + test "SHA1" do + set_content_hash_algorithm(:sha) + + body = "hello world" + content_hash = :crypto.hash(:sha, body) |> Base.encode64() + expected = %{"x-amz-checksum-sha1" => content_hash} + assert ExAws.S3.calculate_content_header("hello world") === expected + + on_exit(&unset_content_hash_algorithm/0) + end + test "SHA256" do - set_content_hash_algorithm() + set_content_hash_algorithm(:sha256) body = "hello world" content_hash = :crypto.hash(:sha256, body) |> Base.encode64() expected = %{"x-amz-checksum-sha256" => content_hash} assert ExAws.S3.calculate_content_header("hello world") === expected - unset_content_hash_algorithm() + on_exit(&unset_content_hash_algorithm/0) end end - defp set_content_hash_algorithm() do - Application.put_env(:ex_aws_s3, :content_hash_algorithm, :sha256) + defp set_content_hash_algorithm(alg) do + Application.put_env(:ex_aws_s3, :content_hash_algorithm, alg) end defp unset_content_hash_algorithm() do From f16e15e75ce68043df0983e3d33293af251e673f Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Wed, 24 Jan 2024 09:36:40 -0500 Subject: [PATCH 4/6] Add additional docs about hash config --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 76b628d..732c3e6 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,13 @@ config :ex_aws, :s3, port: 9000 ``` +An alternate `content_hash_algorithm` can be specified as well. The default is `:md5`. It may be necessary to change this when operating in a FIPS-compliant environment where MD5 isn't available, for instance. At this time, only `:sha256` and `:sha` are supported by both Erlang and S3. + +``` elixir +# config.exs +config :ex_aws_s3, :content_hash_algorithm, :sha256 +``` + ## License From d139b6ac6e7333d9722f05563f20852d82434f1f Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Wed, 24 Jan 2024 09:43:43 -0500 Subject: [PATCH 5/6] Update hash algorithm types --- lib/ex_aws/s3.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ex_aws/s3.ex b/lib/ex_aws/s3.ex index 053831b..d9cc484 100644 --- a/lib/ex_aws/s3.ex +++ b/lib/ex_aws/s3.ex @@ -80,12 +80,12 @@ defmodule ExAws.S3 do @type amz_meta_opts :: [{atom, binary} | {binary, binary}, ...] @typedoc """ - The hashing algorithms that S3 supports for content integrity. - Erlang only supports sha1 and sha256. + The hashing algorithms that both S3 and Erlang support. https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + https://www.erlang.org/doc/man/crypto.html#type-hash_algorithm """ - @type hash_algorithm :: :crc32 | :crc32c | :sha1 | :sha256 + @type hash_algorithm :: :sha | :sha256 | :md5 ## Buckets ############# @@ -1014,7 +1014,7 @@ defmodule ExAws.S3 do {hash_header(alg), :crypto.hash(alg, content) |> Base.encode64()} end - @spec get_hash_config() :: atom() + @spec get_hash_config() :: hash_algorithm() defp get_hash_config() do Application.get_env(:ex_aws_s3, :content_hash_algorithm) || :md5 end From 6d4712ae992fea37c51070f518986356de40aea6 Mon Sep 17 00:00:00 2001 From: Andrew Anguiano Date: Wed, 24 Jan 2024 09:46:35 -0500 Subject: [PATCH 6/6] Tweak hash docs for clarity --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 732c3e6..22ffa1e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ config :ex_aws, :s3, port: 9000 ``` -An alternate `content_hash_algorithm` can be specified as well. The default is `:md5`. It may be necessary to change this when operating in a FIPS-compliant environment where MD5 isn't available, for instance. At this time, only `:sha256` and `:sha` are supported by both Erlang and S3. +An alternate `content_hash_algorithm` can be specified as well. The default is `:md5`. It may be necessary to change this when operating in a FIPS-compliant environment where MD5 isn't available, for instance. At this time, only `:sha256`, `:sha`, and `:md5` are supported by both Erlang and S3. ``` elixir # config.exs